diff --git a/.gitattributes b/.gitattributes index d9499238c02481cee95ce39005f4b6352621f686..394fc495e2f262c52be3ea9c7bba89c6832eec1d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -169,3 +169,68 @@ jqiikotd/episode_rosbags/episode_2_2025-01-07-17-25-13.bag filter=lfs diff=lfs m 1evqtpqc/episode_rosbags/episode_9_2025-01-07-23-17-45.bag filter=lfs diff=lfs merge=lfs -text 7u1dchof/episode_rosbags/episode_3_2025-01-07-21-52-25.bag filter=lfs diff=lfs merge=lfs -text 7u1dchof/episode_rosbags/episode_5_2025-01-07-21-53-53.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_6_2025-01-07-21-54-40.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_8_2025-01-07-21-56-07.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/episode_rosbags/episode_3_2025-01-07-17-39-11.bag filter=lfs diff=lfs merge=lfs -text +b3d2fbi3/episode_rosbags/episode_1_2025-01-07-21-45-44.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_14_2025-01-07-22-01-02.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_11_2025-01-07-21-58-27.bag filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_12_2025-01-07-21-59-16.bag filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +8ycpgp3n/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_9_2025-01-07-21-56-51.bag filter=lfs diff=lfs merge=lfs -text +b3d2fbi3/episode_rosbags/episode_2_2025-01-07-21-46-30.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_4_2025-01-07-21-53-10.bag filter=lfs diff=lfs merge=lfs -text +7u1dchof/episode_rosbags/episode_1_2025-01-07-21-50-43.bag filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kccehv5k/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag filter=lfs diff=lfs merge=lfs -text diff --git a/1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag b/1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..ce28ab5d53953c03067b9ed7f2270df903a26272 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f85c6bd8563ea9b56f64ea14d4e46057cb65ee627df3425073dc437a143c23 +size 1407602106 diff --git a/28myr1bk/.hydra/config.yaml b/28myr1bk/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/28myr1bk/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/28myr1bk/.hydra/hydra.yaml b/28myr1bk/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f6109b11bd1ad81a416f70cb4fb56b645999bb5 --- /dev/null +++ b/28myr1bk/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/28myr1bk + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/28myr1bk/.hydra/overrides.yaml b/28myr1bk/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/28myr1bk/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/28myr1bk/eval_policy.log b/28myr1bk/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..095ab9521085871b79238f5f72d5cdd97668d3d3 --- /dev/null +++ b/28myr1bk/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-31 20:46:25,817][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 20:46:25,820][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-31 20:46:30,096][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/7u1dchof/.hydra/config.yaml b/7u1dchof/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d1c2ea213e59070360e2868dcd64da0e46173d1 --- /dev/null +++ b/7u1dchof/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 15 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7u1dchof/.hydra/hydra.yaml b/7u1dchof/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..416c9ed731ef260aa3cd2ce0ca431d74ca8e005b --- /dev/null +++ b/7u1dchof/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7u1dchof/.hydra/overrides.yaml b/7u1dchof/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7u1dchof/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7u1dchof/episode_rosbags/color_K.npy b/7u1dchof/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/7u1dchof/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/7u1dchof/episode_rosbags/episode_11_2025-01-07-21-58-27.bag b/7u1dchof/episode_rosbags/episode_11_2025-01-07-21-58-27.bag new file mode 100644 index 0000000000000000000000000000000000000000..3c1b341bcd77368f2913adce15a4a785fddcdfb6 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_11_2025-01-07-21-58-27.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c87ed7e47b8cb61c2e65819c13a8f84e4655646b846cbdb0d72fb2d2ab3f2c +size 1412741291 diff --git a/7u1dchof/episode_rosbags/episode_12_2025-01-07-21-59-16.bag b/7u1dchof/episode_rosbags/episode_12_2025-01-07-21-59-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..f1720c90afd0607604bcbb34998c4d62aa22ae93 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_12_2025-01-07-21-59-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ae13eea17d77d88decf8eb830e9e6d701851110824a458b054c5dc710971d0 +size 1413610276 diff --git a/7u1dchof/episode_rosbags/episode_14_2025-01-07-22-01-02.bag b/7u1dchof/episode_rosbags/episode_14_2025-01-07-22-01-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..2f6c3a469a3cc6f9a519e890379cbee13beac173 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_14_2025-01-07-22-01-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5e457e195b4f47d19874b780862d7ecd71b79bb612046ee910bd13726936eb +size 1411052479 diff --git a/7u1dchof/episode_rosbags/episode_1_2025-01-07-21-50-43.bag b/7u1dchof/episode_rosbags/episode_1_2025-01-07-21-50-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..af96135f950db4a6a90b408d127b3a43db6b650c --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_1_2025-01-07-21-50-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7bc572bda83f7a1bc6c1b53379463f159d5d34e691fef025815fc03e44f3e5 +size 1410626651 diff --git a/7u1dchof/episode_rosbags/episode_4_2025-01-07-21-53-10.bag b/7u1dchof/episode_rosbags/episode_4_2025-01-07-21-53-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..7c604ab068a6824869bea420d904960b6f62b5b6 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_4_2025-01-07-21-53-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c75f5129f65730c0a32b5c3fa2f178209a733a6c8e35689812b0c3458fe88cd +size 1407699511 diff --git a/7u1dchof/episode_rosbags/episode_6_2025-01-07-21-54-40.bag b/7u1dchof/episode_rosbags/episode_6_2025-01-07-21-54-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..6fb753246c6a52d7dc5c4664e00d5337a52859fa --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_6_2025-01-07-21-54-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489abb16cac8e0d5c3eaffe8267d336a907e027cf3cd58806e0afd4fa77ff791 +size 1413616814 diff --git a/7u1dchof/episode_rosbags/episode_8_2025-01-07-21-56-07.bag b/7u1dchof/episode_rosbags/episode_8_2025-01-07-21-56-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..aa5545052441b7c2667b1f439af627e3b07f81b9 --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_8_2025-01-07-21-56-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b74333b45a5a6c7fb01d74b9e21b8a886c44a4e2650520a1e60f0f66d0815f +size 1410720372 diff --git a/7u1dchof/episode_rosbags/episode_9_2025-01-07-21-56-51.bag b/7u1dchof/episode_rosbags/episode_9_2025-01-07-21-56-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..5d4eeb96f5a3e15f370c73111509fcea49ba394c --- /dev/null +++ b/7u1dchof/episode_rosbags/episode_9_2025-01-07-21-56-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da6fa7f4eac2126a3f140821613da9a369fd62d106f97da7741c32af904606be +size 1415554756 diff --git a/7u1dchof/eval_video/0_eval.mp4 b/7u1dchof/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3ce324ce47b57f6760d2fdf7a03869b2c23d726 --- /dev/null +++ b/7u1dchof/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165af090a1eeabfb388bdb91193faa030692627f90627267d80076485db84af2 +size 1198949 diff --git a/7u1dchof/eval_video/10_eval.mp4 b/7u1dchof/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38eb830ba1c973c5451eb635756b2fc3b707b460 --- /dev/null +++ b/7u1dchof/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc846afee20f9965cbd4c9c5df83d573e6400ec5460c50cd9b53968e95526f48 +size 1350850 diff --git a/7u1dchof/eval_video/11_eval.mp4 b/7u1dchof/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4cbae04b08116575614a2d687780b3bdf386c63d --- /dev/null +++ b/7u1dchof/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac8f9c871b47a881b7078cf387c6f932e510097420cba7f35847ae6d6af4fc4 +size 1343960 diff --git a/7u1dchof/eval_video/12_eval.mp4 b/7u1dchof/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d358aa745a58e6acda59303bf28098a8066ec74 --- /dev/null +++ b/7u1dchof/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d8d04ea7edc7b1c818d56fc72dd6a7fb05ef251ba2c98faa900c65a799918a +size 1372727 diff --git a/7u1dchof/eval_video/13_eval.mp4 b/7u1dchof/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1cafc50b12080ae7b13e4d815175a70ce82d52c --- /dev/null +++ b/7u1dchof/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac1e3a9ea31f7ab5b3ff57bc4ef9121c87cddfe2da60f3b4316d84ac95905db +size 1449690 diff --git a/7u1dchof/eval_video/14_eval.mp4 b/7u1dchof/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f14ec23d34abb778ce12af05f392de08c262231a --- /dev/null +++ b/7u1dchof/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dac4d6100436c1334e46d2831ed52c015b258163d6956a4fc34400b4f558a8b +size 1395817 diff --git a/7u1dchof/eval_video/1_eval.mp4 b/7u1dchof/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb9ec7d9d1a236ae4e1154abb7dcf2aacfdd5a0b --- /dev/null +++ b/7u1dchof/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4095f09212ed4cd67c65f8775628ad42b68590c26b78ab328b3e7265cdce459 +size 1207724 diff --git a/7u1dchof/eval_video/2_eval.mp4 b/7u1dchof/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77b3bfd39a556dd78cdb60ae47fc440525bddf13 --- /dev/null +++ b/7u1dchof/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a3689635b40f3325683ba3e3cb26e079f0b0afe9adc75b43296b68b24b8af4 +size 1216523 diff --git a/7u1dchof/eval_video/3_eval.mp4 b/7u1dchof/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..027c5d7a1fe9fc5354c2f62ef315a80ece109742 --- /dev/null +++ b/7u1dchof/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209651fa62317471e2aef900bc59c84ea481b2c3cbcde1a3c9f2109b6fa9d496 +size 1219522 diff --git a/7u1dchof/eval_video/4_eval.mp4 b/7u1dchof/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40234c60c8d109f0edd32273f595237429cf6dba --- /dev/null +++ b/7u1dchof/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ae6c438be885a5e86e7108b0d37c1e0b43a0186c90c098bd710d9fc3afaf56 +size 1205814 diff --git a/7u1dchof/eval_video/5_eval.mp4 b/7u1dchof/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..701cb7498fb9404cf8d6fb45c910d43ccb9a5003 --- /dev/null +++ b/7u1dchof/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c374bcb2343831d6c96aae9c50249b7f50652c6eecd1c5aa2b3e58eb9e1948c +size 1198973 diff --git a/7u1dchof/eval_video/6_eval.mp4 b/7u1dchof/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7dc5969a428b8ac5d1a8aa244edee3d949380b7 --- /dev/null +++ b/7u1dchof/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fd0a12988dce00dfb925d718a3a1c0a1fe9730fcaed1281e9590f59aaf3cdd5 +size 1263632 diff --git a/7u1dchof/eval_video/7_eval.mp4 b/7u1dchof/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5233eb380f6c32061702b165f0621629b8bfd885 --- /dev/null +++ b/7u1dchof/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ba49b1c68cf3a00e67a25fde51a6f248444542077115544ccbb748a2daa6b56 +size 1209461 diff --git a/7u1dchof/eval_video/8_eval.mp4 b/7u1dchof/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..604f278ab3026ef34575d01d1856e45ec1c451f0 --- /dev/null +++ b/7u1dchof/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2cf0a27b6e7a9a0aea7fd2ff959017bd547f0147819f4f53da4571dd46037a4 +size 1214532 diff --git a/7u1dchof/eval_video/9_eval.mp4 b/7u1dchof/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65584242ec2e8226bbbcb8f4ca4805d409bd54ea --- /dev/null +++ b/7u1dchof/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda3ab83a50fa0b2dcdcf5ad8b124e14ebc8cef3e14800250c2afb513a023ef5 +size 1320353 diff --git a/7u1dchof/tb/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 b/7u1dchof/tb/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 new file mode 100644 index 0000000000000000000000000000000000000000..b731a1dea2ff78605290e3ec874af768d1dd0aa6 --- /dev/null +++ b/7u1dchof/tb/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9cc284870066b8f27c1998e346f2eaa1879ff3668bf00e41da049789cf9239d +size 17369 diff --git a/7u1dchof/wandb/debug-internal.log b/7u1dchof/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5babf183aeb3be59bed481092c884573266192af --- /dev/null +++ b/7u1dchof/wandb/debug-internal.log @@ -0,0 +1,1686 @@ +2025-01-07 21:49:39,046 INFO StreamThr :2237540 [internal.py:wandb_internal():85] W&B internal server running at pid: 2237540, started at: 2025-01-07 21:49:39.046048 +2025-01-07 21:49:39,047 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status +2025-01-07 21:49:39,061 INFO WriterThread:2237540 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb +2025-01-07 21:49:39,062 DEBUG SenderThread:2237540 [sender.py:send():379] send: header +2025-01-07 21:49:39,069 DEBUG SenderThread:2237540 [sender.py:send():379] send: run +2025-01-07 21:49:39,302 INFO SenderThread:2237540 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files +2025-01-07 21:49:39,302 INFO SenderThread:2237540 [sender.py:_start_run_threads():1188] run started: 7u1dchof with start time 1736304579.046338 +2025-01-07 21:49:39,316 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 21:49:39,316 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: check_version +2025-01-07 21:49:39,361 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 21:49:39,366 DEBUG HandlerThread:2237540 [system_info.py:__init__():26] System info init +2025-01-07 21:49:39,366 DEBUG HandlerThread:2237540 [system_info.py:__init__():41] System info init done +2025-01-07 21:49:39,366 INFO HandlerThread:2237540 [system_monitor.py:start():194] Starting system monitor +2025-01-07 21:49:39,366 INFO SystemMonitor:2237540 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 21:49:39,366 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 21:49:39,367 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started disk monitoring +2025-01-07 21:49:39,367 INFO HandlerThread:2237540 [system_monitor.py:probe():214] Collecting system info +2025-01-07 21:49:39,367 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 21:49:39,368 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started memory monitoring +2025-01-07 21:49:39,368 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started network monitoring +2025-01-07 21:49:39,378 DEBUG HandlerThread:2237540 [system_info.py:probe():152] Probing system +2025-01-07 21:49:39,380 DEBUG HandlerThread:2237540 [system_info.py:_probe_git():137] Probing git +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:_probe_git():145] Probing git done +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:probe():200] Probing system done +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T02:49:39.378676', 'startedAt': '2025-01-08T02:49:39.039301', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3558.6397500000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4485.477, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.04, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 637.9640197753906}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 21:49:39,384 INFO HandlerThread:2237540 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 21:49:39,384 INFO HandlerThread:2237540 [system_monitor.py:probe():227] Publishing system info +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:_save_code():44] Saving code +2025-01-07 21:49:39,386 DEBUG HandlerThread:2237540 [system_info.py:_save_code():67] Saving code done +2025-01-07 21:49:39,386 DEBUG HandlerThread:2237540 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 21:49:39,451 DEBUG HandlerThread:2237540 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 21:49:39,451 INFO HandlerThread:2237540 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 21:49:39,454 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:49:39,454 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 21:49:39,455 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 21:49:39,455 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 21:49:39,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 21:49:39,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:49:39,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: python_packages +2025-01-07 21:49:39,694 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ohqyrws2-wandb-metadata.json +2025-01-07 21:49:39,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:49:39,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:39,697 INFO HandlerThread:2237540 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/tb" + save: true +} +control { + relay_id: "b280c0d6940a4957b38b21bf92a79360" +} +_info { + stream_id: "7u1dchof" +} + +2025-01-07 21:49:39,697 DEBUG HandlerThread:2237540 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 21:49:39,705 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/k05q5dgz-diff.patch +2025-01-07 21:49:39,760 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ukefgs7x-code/FISH/eval_robot.py +2025-01-07 21:49:39,797 DEBUG SenderThread:2237540 [sender.py:send():379] send: telemetry +2025-01-07 21:49:39,798 DEBUG SenderThread:2237540 [sender.py:send():379] send: tbrecord +2025-01-07 21:49:39,798 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:49:39,798 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 with policy live +2025-01-07 21:49:40,017 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lr08xa1p-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:49:40,303 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH +2025-01-07 21:49:40,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:41,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:42,303 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:42,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:43,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:44,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:44,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:45,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:46,481 DEBUG SenderThread:2237540 [sender.py:send():379] send: config +2025-01-07 21:49:46,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:47,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:48,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:48,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:49,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:49,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:50,305 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:50,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:51,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:52,306 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:52,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:53,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:54,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:49:54,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:49:54,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:54,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:55,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:56,307 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:56,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:57,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:58,307 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:58,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:59,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:00,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:00,308 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:00,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:01,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:02,309 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:02,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:03,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:04,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:05,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:05,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:06,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:07,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:08,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:09,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:09,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:10,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:10,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:11,311 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 21:50:11,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:12,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:13,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:14,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:15,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:16,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:16,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:17,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:18,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:19,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:20,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:21,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:21,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:22,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:23,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:24,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:24,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:24,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:25,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:26,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:26,773 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:27,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:28,162 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:50:28,162 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 with policy now +2025-01-07 21:50:28,162 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:50:28,163 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:50:28,163 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:50:28,163 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:50:28,315 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos +2025-01-07 21:50:28,461 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lwao7e3p-media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:28,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:28,742 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/r1mxz7xl-media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:29,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:30,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:30,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:31,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:32,621 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:32,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:33,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:34,317 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:34,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:35,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:36,318 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:36,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:37,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:37,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:38,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:39,368 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 21:50:39,369 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:50:39,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:39,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:39,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:40,319 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:50:40,588 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/pvxw69ed-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:50:40,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:41,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:42,320 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:42,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:43,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:43,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:44,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:45,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:46,321 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:46,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:47,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:48,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:48,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:49,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:50,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:51,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:52,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:53,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:53,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:54,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:54,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:54,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:55,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:56,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:57,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:58,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:58,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:59,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:00,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:01,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:02,326 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:02,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:03,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:03,858 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:04,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:05,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:06,328 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:06,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:07,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:08,328 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:08,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:09,253 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:09,370 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:51:09,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:09,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:10,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:11,708 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:12,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:51:12,085 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:51:12,086 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 with policy now +2025-01-07 21:51:12,086 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:51:12,086 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:51:12,086 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:51:12,347 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7w50oiu3-media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,615 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/smxnqzhc-media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:13,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:14,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:14,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:14,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:15,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:16,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:17,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:18,332 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:18,712 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:19,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:19,712 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:20,333 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:20,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:21,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:22,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:23,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:24,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:24,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:24,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:24,818 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:25,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:26,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:27,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:28,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:29,336 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:51:29,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:30,125 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:51:30,126 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:51:30,126 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:30,126 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:51:30,336 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:30,337 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:51:30,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:31,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:32,337 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:32,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:33,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:34,338 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:34,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:35,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:35,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:36,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:37,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:38,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:39,370 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:51:39,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:39,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:39,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:40,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:40,816 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:41,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:42,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:43,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:44,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:45,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:46,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:46,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:47,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:48,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:49,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:50,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:51,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:52,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:52,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:53,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:54,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:54,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:54,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:55,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:56,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:57,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:57,834 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:58,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:59,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:00,725 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:01,545 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:52:01,545 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 with policy now +2025-01-07 21:52:01,549 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:52:01,549 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:01,550 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:01,551 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:01,721 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:01,810 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/u6icwa1c-media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:52:02,660 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/bwtpnajq-media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:03,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:03,573 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:03,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:04,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:04,728 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:05,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:06,349 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:06,728 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:07,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:08,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:08,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:09,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:52:09,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:09,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:10,350 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:10,731 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:11,736 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:12,734 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:13,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:14,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:14,736 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:15,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:16,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:17,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:18,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:19,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:20,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:20,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:21,354 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:52:21,606 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8p4tfv7o-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:52:21,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:21,946 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:21,946 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:21,946 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:22,354 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:22,355 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:22,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:23,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:24,355 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:24,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:24,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:24,748 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:25,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:26,356 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:26,495 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:26,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:27,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:28,357 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:28,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:29,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:30,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:31,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:31,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:32,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:33,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:34,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:35,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:36,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:37,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:37,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:38,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:39,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:52:39,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:39,696 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:39,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:40,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:41,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:42,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:42,811 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:43,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:44,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:45,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:46,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:47,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:48,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:48,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:49,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:50,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:51,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:52,748 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:53,606 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:52:53,606 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 with policy now +2025-01-07 21:52:53,609 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:52:53,610 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:53,610 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:53,610 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:53,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:53,870 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/wqvd6pty-media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,366 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:52:54,639 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:54,662 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/c9y1qb0c-media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:54,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:54,753 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:55,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:55,755 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:56,366 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:56,760 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:57,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:58,367 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:58,757 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:59,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:59,756 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:00,752 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:01,753 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:02,368 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:02,755 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:03,758 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:04,758 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:05,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:05,770 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:06,369 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:06,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:07,004 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:07,004 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:07,005 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:07,370 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:07,769 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:08,370 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:08,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:09,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:53:09,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:09,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:09,767 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:10,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:11,506 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:11,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:12,371 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:12,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:13,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:14,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:15,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:16,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:16,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:17,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:18,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:19,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:20,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:21,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:21,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:22,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:23,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:24,699 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:24,699 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:24,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:25,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:26,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:26,837 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:27,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:28,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:29,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:30,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:31,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:32,708 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:32,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:33,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:34,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:35,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:36,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:37,768 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:38,004 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:38,384 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:38,692 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:53:38,692 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 with policy now +2025-01-07 21:53:38,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:53:38,697 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:38,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:38,698 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:38,771 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:38,949 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/k3ukfryr-media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,372 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:53:39,675 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7c94hrg0-media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:39,777 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:40,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:40,774 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:41,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:42,782 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:43,077 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:43,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:44,386 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:44,788 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:45,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:46,387 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:46,784 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:47,783 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:48,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:48,870 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:49,788 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:50,056 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:50,057 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:50,057 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:50,648 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8k5znsay-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:50,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:51,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:52,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:52,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:53,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:54,390 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:54,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:54,700 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:54,781 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:54,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:55,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:56,391 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:56,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:57,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:58,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:59,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:00,392 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:00,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:00,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:01,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:02,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:03,792 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:04,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:05,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:05,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:06,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:07,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:08,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:09,372 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:54:09,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:09,700 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:09,794 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:10,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:10,850 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:11,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:12,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:13,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:14,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:15,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:16,711 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:16,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:17,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:18,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:19,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:20,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:21,677 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:54:21,677 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 with policy now +2025-01-07 21:54:21,683 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:54:21,684 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:54:21,684 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:54:21,684 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:54:21,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:21,807 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:21,936 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/2l8065zn-media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:54:22,686 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/crb62mbz-media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:23,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:24,405 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:24,711 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:24,711 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:24,803 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:25,812 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:26,405 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:26,806 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:27,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:27,807 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:28,808 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:29,809 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:30,407 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:30,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:31,815 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:32,716 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:32,816 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:33,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:34,818 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:35,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:36,112 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:54:36,112 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:54:36,113 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:54:36,409 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:54:36,409 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:54:36,828 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:37,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:38,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:38,410 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:38,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:39,373 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:54:39,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:39,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:40,825 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:41,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:42,411 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:42,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:43,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:44,174 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:44,412 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:44,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:45,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:46,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:47,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:48,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:49,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:49,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:50,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:51,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:52,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:53,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:54,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:54,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:54,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:54,827 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:55,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:56,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:57,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:58,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:59,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:00,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:00,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:01,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:02,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:03,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:04,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:05,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:06,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:06,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:07,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:08,421 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:08,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:09,373 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:55:09,722 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:09,722 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:09,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:55:09,824 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:55:09,824 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 with policy now +2025-01-07 21:55:09,824 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:09,825 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:09,825 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:09,825 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:10,083 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/r120ktgi-media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,421 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:55:10,699 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/tbzwhest-media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,833 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:11,826 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:11,828 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:12,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:12,841 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:13,849 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:14,423 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:14,849 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:15,852 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:16,851 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:17,161 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:17,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:18,425 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:18,850 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:19,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:20,167 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:20,168 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:20,168 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:20,426 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:55:20,426 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:20,682 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lhd51kd1-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:55:20,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:21,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:22,233 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:22,427 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:22,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:23,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:24,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:24,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:24,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:25,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:26,429 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:26,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:27,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:27,922 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:28,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:29,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:30,429 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:30,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:31,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:32,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:33,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:33,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:34,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:35,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:36,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:37,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:38,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:38,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:39,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:55:39,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:39,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:40,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:41,861 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:42,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:43,803 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:43,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:44,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:45,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:46,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:47,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:48,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:49,721 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:49,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:50,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:51,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:52,437 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:52,478 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:55:52,478 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 with policy now +2025-01-07 21:55:52,482 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:55:52,483 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:52,483 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:52,484 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:52,735 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/tz4dzcsu-media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:52,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:55:53,739 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/jrod8liv-media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:53,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:54,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:54,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:54,709 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:54,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:54,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:55,874 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:56,888 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:57,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:58,440 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:58,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:59,878 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:00,440 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:00,738 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:00,878 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:01,879 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:02,877 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:03,881 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:04,224 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:04,224 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:04,225 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:04,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:04,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:04,443 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:04,881 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:05,900 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:06,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:06,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:06,891 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:07,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:08,443 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:08,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:09,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:56:09,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:09,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:09,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:10,444 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:10,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:11,842 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:11,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:12,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:13,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:14,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:15,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:16,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:17,725 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:17,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:18,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:19,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:20,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:21,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:22,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:23,723 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:23,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:24,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:24,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:24,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:25,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:26,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:27,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:28,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:28,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:29,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:30,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:31,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:32,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:33,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:34,532 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:34,901 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:35,771 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:56:35,772 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 with policy now +2025-01-07 21:56:35,786 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:56:35,787 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:35,787 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:35,787 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:35,902 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:36,028 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8oizt1zu-media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:56:36,777 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/p0lzpr9w-media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,906 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:37,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:38,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:38,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:39,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:56:39,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:39,730 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:39,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:39,913 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:40,455 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:40,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:41,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:42,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:43,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:44,456 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:44,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:45,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:45,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:46,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:47,927 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:48,279 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:48,279 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:48,280 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:48,458 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:48,458 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:48,714 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/2won4f61-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:48,939 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:49,942 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:50,459 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:50,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:50,943 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:51,941 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:52,955 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:53,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:54,460 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:54,734 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:54,734 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:54,951 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:55,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:55,961 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:56,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:57,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:58,462 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:58,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:59,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:00,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:01,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:01,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:02,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:03,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:04,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:05,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:06,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:07,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:07,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:08,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:09,375 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:57:09,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:09,718 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:09,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:10,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:11,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:12,786 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:12,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:13,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:14,468 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:14,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:15,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:16,469 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:16,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:17,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:18,521 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:18,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:19,950 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:20,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:21,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:22,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:23,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:23,950 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:24,472 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:24,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:24,740 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:24,953 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:25,132 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:57:25,132 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 with policy now +2025-01-07 21:57:25,138 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:57:25,138 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:57:25,139 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:57:25,139 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:57:25,394 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/rs8mej0e-media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:57:25,759 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/3gxbrza1-media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,964 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:26,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:26,970 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:27,959 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:28,474 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:28,799 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:28,997 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:29,983 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:30,474 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:30,991 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:31,979 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:32,476 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:32,992 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:33,993 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:34,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:34,984 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:36,001 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:36,333 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:57:36,333 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:57:36,334 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:57:36,477 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:57:36,477 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:36,478 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:57:37,017 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:38,009 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:38,478 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:39,012 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:39,375 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:57:39,729 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:39,729 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:39,874 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:40,018 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:40,479 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:41,036 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:42,011 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:43,011 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:44,017 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:44,485 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:45,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:45,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:46,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:46,486 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:47,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:48,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:49,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:50,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:51,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:51,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:52,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:53,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:54,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:54,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:54,719 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:55,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:56,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:56,864 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:57,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:58,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:59,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:00,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:01,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:02,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:02,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:03,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:04,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:05,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:06,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:07,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:07,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:08,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:09,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:09,376 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:58:09,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:09,719 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:10,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:11,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:12,044 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:12,495 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:12,661 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:58:12,662 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 with policy now +2025-01-07 21:58:12,662 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:58:12,662 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:58:12,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:58:12,663 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:58:12,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:12,913 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/yv4ldzu5-media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:13,051 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:58:13,796 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7o9ep2ct-media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:14,051 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:14,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:15,072 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:16,065 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:17,073 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:18,071 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:18,173 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:18,498 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:19,072 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:20,081 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:20,499 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:21,109 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:22,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:23,076 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:23,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:24,102 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:24,394 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:58:24,394 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:58:24,394 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:58:24,500 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:58:24,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:24,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:58:24,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:24,730 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:24,763 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/sdfhlm89-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:58:25,104 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:26,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:26,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:27,087 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:28,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:28,502 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:28,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:29,097 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:30,101 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:30,502 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:31,094 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:32,105 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:32,504 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:33,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:34,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:34,247 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:34,505 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:35,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:36,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:36,506 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:37,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:38,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:38,506 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:39,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:39,376 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:58:39,377 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:39,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:39,724 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:40,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:40,507 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:41,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:42,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:42,508 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:43,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:44,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:44,488 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:45,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:46,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:47,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:48,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:48,512 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:49,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:49,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:50,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:50,513 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:51,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:52,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:53,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:54,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:54,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:54,724 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:54,834 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:55,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:56,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:57,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:58,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:59,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:59,877 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:00,103 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:00,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:00,643 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:59:00,643 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 with policy now +2025-01-07 21:59:00,652 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:59:00,653 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:00,653 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:00,653 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:00,902 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/h2bw89nv-media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:01,091 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:59:01,828 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/l7gc7soe-media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:02,096 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:02,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:03,104 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:04,129 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:05,122 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:05,773 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:06,114 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:06,520 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:07,123 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:08,132 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:08,521 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:09,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:09,377 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:59:09,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:09,738 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:10,127 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:10,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:11,141 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:12,123 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:12,464 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:12,464 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:12,464 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:12,522 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:59:12,522 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:12,523 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:13,125 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:14,137 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:14,523 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:15,133 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:16,031 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:16,132 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:17,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:18,162 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:18,525 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:19,150 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:20,149 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:21,176 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:21,777 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:22,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:23,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:24,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:24,527 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:24,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:24,725 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:25,135 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:26,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:26,847 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:27,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:28,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:29,135 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:30,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:31,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:32,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:32,766 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:33,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:34,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:35,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:36,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:36,531 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:37,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:37,766 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:38,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:39,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:39,377 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:59:39,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:39,725 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:40,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:41,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:42,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:42,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:43,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:44,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:45,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:46,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:46,539 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:47,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:48,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:48,225 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:49,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:50,171 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:50,541 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:50,665 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:59:50,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 with policy now +2025-01-07 21:59:50,669 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:59:50,669 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:50,670 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:50,670 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:50,930 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/67y65hit-media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:51,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:51,541 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:51,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:51,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:59:51,874 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/4edgdnp9-media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:52,179 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:52,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:53,198 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:53,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:54,197 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:54,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:54,746 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:55,215 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:56,199 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:56,543 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:57,225 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:58,224 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:58,544 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:59,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:59,581 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:00,235 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:01,213 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:02,217 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:02,520 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:02,520 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:02,521 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:00:03,215 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:04,201 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:04,547 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:04,814 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:05,244 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:06,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:07,214 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:08,218 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:08,548 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:09,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:09,378 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:00:09,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:09,739 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:09,894 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:10,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:11,233 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:12,232 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:13,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:14,220 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:14,551 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:15,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:15,799 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:16,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:17,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:18,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:19,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:20,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:20,801 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:21,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:22,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:23,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:24,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:24,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:24,732 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:25,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:25,863 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:26,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:27,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:28,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:29,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:30,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:31,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:31,800 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:32,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:33,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:34,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:35,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:36,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:36,800 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:37,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:38,224 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:39,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:39,378 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:00:39,765 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:39,765 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:40,245 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:40,561 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:41,022 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:00:41,023 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 with policy now +2025-01-07 22:00:41,023 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:00:41,023 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:41,023 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:41,024 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:41,231 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:41,279 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/9wqjt18f-media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:41,561 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:00:41,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:41,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 22:00:41,850 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ta0ravul-media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:42,131 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:42,238 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:42,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:43,245 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:44,238 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:45,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:46,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:46,563 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:47,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:47,628 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:48,265 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:48,564 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:49,290 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:50,286 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:51,280 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:52,294 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:52,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:53,283 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:54,281 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:54,749 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:54,749 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:55,304 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:56,291 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:57,290 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:58,283 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:58,568 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:58,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:58,595 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:58,595 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:58,595 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:58,595 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:58,823 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/3zwgkeg1-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:59,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:59,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:00,310 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:00,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:01,320 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:02,293 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:02,570 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:03,295 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:04,053 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:04,304 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:05,308 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:06,315 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:07,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:08,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:08,573 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:09,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:09,379 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:01:09,379 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:09,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:09,745 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:10,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:11,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:12,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:13,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:14,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:14,809 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:15,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:17,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:18,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:19,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:19,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:20,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:21,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:22,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:23,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:24,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:24,579 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:24,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:24,745 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:24,898 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:25,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:26,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:27,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:28,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:29,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:30,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:30,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:31,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:32,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:33,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:34,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:35,307 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:35,672 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:35,672 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 with policy now +2025-01-07 22:01:35,682 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:35,682 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:35,682 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:35,682 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:35,939 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/d31nmdix-media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:36,357 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 22:01:36,776 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:36,893 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ib39peo4-media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:37,348 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:38,342 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:38,585 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:39,346 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:39,379 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:01:39,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:39,751 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:40,361 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:40,585 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:41,388 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:41,893 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:42,409 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:43,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:44,350 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:44,587 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:45,357 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:46,351 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:46,588 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:01:46,588 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:46,652 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:46,653 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:46,653 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:47,362 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:47,488 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:01:47,488 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:01:47,591 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:47,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:47,959 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxqm9j9pt +2025-01-07 22:01:48,268 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:01:48,277 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:48,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:48,399 INFO SenderThread:2237540 [sender.py:send_request_log_artifact():1525] logged artifact run-7u1dchof-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjE3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg3ODU0NA==', 'latestArtifact': None}} +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:48,400 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json with policy now +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,401 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:48,401 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,401 DEBUG SenderThread:2237540 [sender.py:send():379] send: telemetry +2025-01-07 22:01:48,401 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,402 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,402 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,402 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,402 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,404 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,404 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,405 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,405 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,405 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,405 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,406 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,406 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,406 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,406 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,407 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,407 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,407 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,408 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,408 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,408 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,409 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,409 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:01:48,589 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/sft7io99-media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,592 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval +2025-01-07 22:01:48,836 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/1mfgcnas-media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:48,843 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/gbfljdtf-media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,854 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxm4ae1ed +2025-01-07 22:01:49,273 INFO SenderThread:2237540 [sender.py:send_request_log_artifact():1525] logged artifact run-7u1dchof-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjE5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg3ODU0OA==', 'latestArtifact': None}} +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:49,273 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json with policy now +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:49,274 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:49,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:49,533 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/gcn6k838-media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:49,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:50,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:50,594 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:51,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:52,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:53,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:53,835 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:54,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:54,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:54,747 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:55,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:56,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:57,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:58,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:58,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:59,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:00,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:01,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:02,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:03,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:04,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:04,371 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:04,413 DEBUG SenderThread:2237540 [sender.py:send():379] send: exit +2025-01-07 22:02:04,413 INFO SenderThread:2237540 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_exit():588] handling runtime: 745 +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_exit():594] send defer +2025-01-07 22:02:04,414 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,414 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 22:02:04,414 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 1 +2025-01-07 22:02:04,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 22:02:04,415 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 2 +2025-01-07 22:02:04,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 22:02:04,415 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 22:02:04,415 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 22:02:04,416 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 22:02:04,469 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 22:02:04,469 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 22:02:04,470 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined network monitor +2025-01-07 22:02:04,470 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,470 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 22:02:04,470 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 3 +2025-01-07 22:02:04,470 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:02:04,470 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,470 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 22:02:04,471 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,471 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 22:02:04,471 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 4 +2025-01-07 22:02:04,471 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,471 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 22:02:04,600 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:06,601 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:09,602 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 22:02:10,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,662 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 22:02:10,662 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 5 +2025-01-07 22:02:10,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:10,663 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:02:10,663 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:02:10,663 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,663 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:10,665 DEBUG SenderThread:2237540 [sender.py:send():379] send: summary +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:10,665 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 6 +2025-01-07 22:02:10,665 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,666 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 22:02:10,666 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 7 +2025-01-07 22:02:10,666 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:10,666 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,666 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 22:02:10,666 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 22:02:11,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:11,603 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:13,835 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 8 +2025-01-07 22:02:13,835 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:13,836 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:13,836 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 22:02:13,836 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:13,836 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 22:02:13,836 INFO SenderThread:2237540 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 22:02:13,839 INFO SenderThread:2237540 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 22:02:13,840 INFO SenderThread:2237540 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 22:02:13,859 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 9 +2025-01-07 22:02:13,859 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:13,859 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 22:02:13,859 DEBUG SenderThread:2237540 [sender.py:send():379] send: artifact +2025-01-07 22:02:14,416 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:14,541 INFO wandb-upload_1:2237540 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmph0mz9co_ +2025-01-07 22:02:14,605 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:14,670 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzwxnw11b +2025-01-07 22:02:14,691 INFO wandb-upload_2:2237540 [upload_job.py:push():88] Uploaded file /tmp/tmpods0y954/wandb-job.json +2025-01-07 22:02:15,296 INFO SenderThread:2237540 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 22:02:15,297 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:15,297 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 22:02:15,297 INFO SenderThread:2237540 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml config.yaml +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json wandb-summary.json +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt requirements.txt +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json wandb-metadata.json +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log output.log +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch diff.patch +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 10 +2025-01-07 22:02:15,617 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:15,617 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:15,618 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 22:02:15,618 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:15,618 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 22:02:15,618 INFO SenderThread:2237540 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:02:15,790 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:15,796 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 22:02:15,839 INFO wandb-upload_3:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt +2025-01-07 22:02:15,877 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/egceoa3o-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:02:15,887 INFO wandb-upload_4:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:16,087 INFO Thread-11 (_thread_body):2237540 [sender.py:transition_state():614] send defer: 11 +2025-01-07 22:02:16,087 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,087 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 22:02:16,087 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,087 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 22:02:16,088 INFO SenderThread:2237540 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 22:02:16,090 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 12 +2025-01-07 22:02:16,090 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,090 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 22:02:16,091 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,091 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 22:02:16,091 INFO SenderThread:2237540 [file_stream.py:finish():601] file stream finish called +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [file_stream.py:finish():605] file stream finish is done +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 13 +2025-01-07 22:02:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,301 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 22:02:16,301 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 14 +2025-01-07 22:02:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,301 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send():379] send: final +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send():379] send: footer +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,302 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 22:02:16,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:16,303 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: server_info +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 22:02:16,305 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 22:02:16,305 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:16,363 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:16,363 INFO MainThread:2237540 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 22:02:16,364 INFO MainThread:2237540 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 22:02:16,364 INFO MainThread:2237540 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 22:02:16,364 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 22:02:16,364 INFO HandlerThread:2237540 [handler.py:finish():882] shutting down handler +2025-01-07 22:02:17,303 INFO WriterThread:2237540 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [sender.py:finish():1615] shutting down sender +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [file_pusher.py:join():175] waiting for file pusher diff --git a/7u1dchof/wandb/debug.log b/7u1dchof/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b9da1526956ab4525ef5eb9b8cf86a181f8c9fe6 --- /dev/null +++ b/7u1dchof/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Configure stats pid to 2237375 +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/settings +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug.log +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug-internal.log +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():569] calling init triggers +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 15, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof'} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():619] starting backend +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():623] setting up manager +2025-01-07 21:49:39,045 INFO MainThread:2237375 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 21:49:39,046 INFO MainThread:2237375 [wandb_init.py:init():631] backend started and connected +2025-01-07 21:49:39,060 INFO MainThread:2237375 [wandb_init.py:init():720] updated telemetry +2025-01-07 21:49:39,067 INFO MainThread:2237375 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 21:49:39,316 INFO MainThread:2237375 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 21:49:39,354 INFO MainThread:2237375 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 21:49:39,355 INFO MainThread:2237375 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 21:49:39,696 INFO MainThread:2237375 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 21:49:39,697 INFO MainThread:2237375 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/tb, True +2025-01-07 21:49:46,481 INFO MainThread:2237375 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 22:02:17,365 WARNING MsgRouterThr:2237375 [router.py:message_loop():77] message_loop has been closed diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64ff76e617bb5bfcc0a95a98759a008e4aa619ff --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 15 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736304579 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..2bf5ca5168938227dfbbec8407249befb542b0cd --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch @@ -0,0 +1,30 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..b473f0b 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 15 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d3b510152a92e912c21224e6415643c41fccc9b7 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.06666666666666667, 0.0021755193424476253, 0.35215300324535925, 0.007666284316897264, 0.29566386346135354, 0.016987718738360427, 0.23827329283226772, 0.028409004305743977, 0.18168704963136206]]} \ No newline at end of file diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json new file mode 100644 index 0000000000000000000000000000000000000000..fb9f0f3481323980d92c8176d10baa81c1634f26 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[35, 85, 0.4117647058823529, 0.295654282014708, 0.5383982211274597, 0.3164127469429979, 0.5167854353751943, 0.32573482397703757, 0.5000778660178029, 0.33988951894862907, 0.4838198050888951]]} \ No newline at end of file diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3ce324ce47b57f6760d2fdf7a03869b2c23d726 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165af090a1eeabfb388bdb91193faa030692627f90627267d80076485db84af2 +size 1198949 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38eb830ba1c973c5451eb635756b2fc3b707b460 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc846afee20f9965cbd4c9c5df83d573e6400ec5460c50cd9b53968e95526f48 +size 1350850 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4cbae04b08116575614a2d687780b3bdf386c63d --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac8f9c871b47a881b7078cf387c6f932e510097420cba7f35847ae6d6af4fc4 +size 1343960 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5d358aa745a58e6acda59303bf28098a8066ec74 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d8d04ea7edc7b1c818d56fc72dd6a7fb05ef251ba2c98faa900c65a799918a +size 1372727 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1cafc50b12080ae7b13e4d815175a70ce82d52c --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac1e3a9ea31f7ab5b3ff57bc4ef9121c87cddfe2da60f3b4316d84ac95905db +size 1449690 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f14ec23d34abb778ce12af05f392de08c262231a --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dac4d6100436c1334e46d2831ed52c015b258163d6956a4fc34400b4f558a8b +size 1395817 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fb9ec7d9d1a236ae4e1154abb7dcf2aacfdd5a0b --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4095f09212ed4cd67c65f8775628ad42b68590c26b78ab328b3e7265cdce459 +size 1207724 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77b3bfd39a556dd78cdb60ae47fc440525bddf13 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a3689635b40f3325683ba3e3cb26e079f0b0afe9adc75b43296b68b24b8af4 +size 1216523 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..027c5d7a1fe9fc5354c2f62ef315a80ece109742 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209651fa62317471e2aef900bc59c84ea481b2c3cbcde1a3c9f2109b6fa9d496 +size 1219522 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40234c60c8d109f0edd32273f595237429cf6dba --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ae6c438be885a5e86e7108b0d37c1e0b43a0186c90c098bd710d9fc3afaf56 +size 1205814 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..701cb7498fb9404cf8d6fb45c910d43ccb9a5003 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c374bcb2343831d6c96aae9c50249b7f50652c6eecd1c5aa2b3e58eb9e1948c +size 1198973 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7dc5969a428b8ac5d1a8aa244edee3d949380b7 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fd0a12988dce00dfb925d718a3a1c0a1fe9730fcaed1281e9590f59aaf3cdd5 +size 1263632 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5233eb380f6c32061702b165f0621629b8bfd885 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ba49b1c68cf3a00e67a25fde51a6f248444542077115544ccbb748a2daa6b56 +size 1209461 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..604f278ab3026ef34575d01d1856e45ec1c451f0 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2cf0a27b6e7a9a0aea7fd2ff959017bd547f0147819f4f53da4571dd46037a4 +size 1214532 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65584242ec2e8226bbbcb8f4ca4805d409bd54ea --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda3ab83a50fa0b2dcdcf5ad8b124e14ebc8cef3e14800250c2afb513a023ef5 +size 1320353 diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4e839f5b2224c95a6eff52ab1f2bc26cc4c2c913 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log @@ -0,0 +1,808 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736304586.482103]: resetting environment +[INFO] [1736304586.487887]: cleared current plan +[INFO] [1736304586.489338]: moving to home +[INFO] [1736304587.492079]: reached home +[INFO] [1736304587.493370]: reset action history +[INFO] [1736304590.252570]: environment reset +Starting episode 0 +[INFO] [1736304590.254800]: resetting environment +[INFO] [1736304590.258042]: cleared current plan +[INFO] [1736304590.259166]: moving to home +[INFO] [1736304591.261220]: reached home +[INFO] [1736304591.262552]: reset action history +[INFO] [1736304593.970081]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736304596.449156]: resetting environment +[INFO] [1736304596.452710]: cleared current plan +[INFO] [1736304596.453724]: moving to home +[INFO] [1736304597.455947]: reached home +[INFO] [1736304597.457300]: reset action history +[INFO] [1736304600.164826]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_0_2025-01-07-21-50-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:42847/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2237666] +process[print_text-2]: started with pid [2237667] +process[rosbag_record-3]: started with pid [2237668] +started bagging! +[INFO] [1736304627.208461]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736304627.940786]: Stored episode 1. +[INFO] [1736304627.941047]: Saving video... +[INFO] [1736304628.151651]: Video saved! +Starting episode 1 +[INFO] [1736304628.164634]: resetting environment +[INFO] [1736304628.169187]: cleared current plan +[INFO] [1736304628.169410]: moving to home +[INFO] [1736304631.574955]: reached home +[INFO] [1736304631.575716]: reset action history +[INFO] [1736304634.283797]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304640.117562]: resetting environment +[INFO] [1736304640.119813]: cleared current plan +[INFO] [1736304640.119953]: moving to home +[INFO] [1736304641.121281]: reached home +[INFO] [1736304641.121696]: reset action history +[INFO] [1736304643.829149]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_1_2025-01-07-21-50-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:35335/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2237777] +process[print_text-5]: started with pid [2237778] +process[rosbag_record-6]: started with pid [2237802] +started bagging! +For topic gripper_width: timestamp difference is 102029991 for nearest: 1736304660805907202 - target: 1736304660703877211 at idx 230 +For topic gripper_width: timestamp difference is 105966676 for nearest: 1736304664005904783 - target: 1736304663899938107 at idx 249 +For topic gripper_width: timestamp difference is 104405501 for nearest: 1736304666205914595 - target: 1736304666101509094 at idx 249 +[INFO] [1736304670.840863]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736304671.564770]: Stored episode 2. +[INFO] [1736304671.565046]: Saving video... +[INFO] [1736304672.042987]: Video saved! +Starting episode 2 +[INFO] [1736304672.069077]: resetting environment +[INFO] [1736304672.093834]: cleared current plan +[INFO] [1736304672.095461]: moving to home +[INFO] [1736304675.501318]: reached home +[INFO] [1736304675.502909]: reset action history +[INFO] [1736304678.224776]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304689.245489]: resetting environment +[INFO] [1736304689.258142]: cleared current plan +[INFO] [1736304689.258382]: moving to home +[INFO] [1736304690.259985]: reached home +[INFO] [1736304690.260478]: reset action history +[INFO] [1736304692.967856]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_2_2025-01-07-21-51-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:44271/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2237914] +process[print_text-8]: started with pid [2237915] +process[rosbag_record-9]: started with pid [2237916] +started bagging! +[INFO] [1736304719.929443]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736304720.773482]: Stored episode 3. +[INFO] [1736304720.776754]: Saving video... +[WARN] [1736304721.266318]: Plan exhausted +[WARN] [1736304721.308420]: Plan exhausted +[WARN] [1736304721.348449]: Plan exhausted +[WARN] [1736304721.388529]: Plan exhausted +[WARN] [1736304721.429043]: Plan exhausted +[WARN] [1736304721.467037]: Plan exhausted +[WARN] [1736304721.506964]: Plan exhausted +[WARN] [1736304721.547058]: Plan exhausted +[INFO] [1736304721.512022]: Video saved! +Starting episode 3 +[INFO] [1736304721.553335]: resetting environment +[INFO] [1736304721.569846]: cleared current plan +[INFO] [1736304721.572237]: moving to home +[INFO] [1736304724.980523]: reached home +[INFO] [1736304724.981444]: reset action history +[INFO] [1736304727.778278]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304741.141673]: resetting environment +[INFO] [1736304741.163917]: cleared current plan +[INFO] [1736304741.166404]: moving to home +[INFO] [1736304742.173429]: reached home +[INFO] [1736304742.176158]: reset action history +[INFO] [1736304744.908745]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_3_2025-01-07-21-52-25.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:37157/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2238076] +process[print_text-11]: started with pid [2238077] +process[rosbag_record-12]: started with pid [2238078] +started bagging! +[INFO] [1736304772.289696]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736304773.109765]: Stored episode 4. +[INFO] [1736304773.111684]: Saving video... +[WARN] [1736304773.626359]: Plan exhausted +[INFO] [1736304773.574277]: Video saved! +Starting episode 4 +[INFO] [1736304773.611192]: resetting environment +[INFO] [1736304773.637196]: cleared current plan +[INFO] [1736304773.638942]: moving to home +[INFO] [1736304777.049778]: reached home +[INFO] [1736304777.060527]: reset action history +[INFO] [1736304779.867089]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304786.283102]: resetting environment +[INFO] [1736304786.294995]: cleared current plan +[INFO] [1736304786.299118]: moving to home +[INFO] [1736304787.314790]: reached home +[INFO] [1736304787.321989]: reset action history +[INFO] [1736304790.104111]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_4_2025-01-07-21-53-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:41373/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2238210] +process[print_text-14]: started with pid [2238211] +process[rosbag_record-15]: started with pid [2238212] +started bagging! +[INFO] [1736304817.224538]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[WARN] [1736304818.592951]: Plan exhausted +[WARN] [1736304818.627946]: Plan exhausted +[WARN] [1736304818.667605]: Plan exhausted +[WARN] [1736304818.706315]: Plan exhausted +[INFO] [1736304818.004471]: Stored episode 5. +[INFO] [1736304818.009958]: Saving video... +[INFO] [1736304818.642998]: Video saved! +Starting episode 5 +[INFO] [1736304818.702040]: resetting environment +[INFO] [1736304818.709199]: cleared current plan +[INFO] [1736304818.714227]: moving to home +[INFO] [1736304822.032969]: reached home +[INFO] [1736304822.035522]: reset action history +[INFO] [1736304824.822977]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304829.616410]: resetting environment +[INFO] [1736304829.643063]: cleared current plan +[INFO] [1736304829.644727]: moving to home +[INFO] [1736304830.657651]: reached home +[INFO] [1736304830.661365]: reset action history +[INFO] [1736304833.439600]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_5_2025-01-07-21-53-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:45793/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2238323] +process[print_text-17]: started with pid [2238324] +process[rosbag_record-18]: started with pid [2238348] +started bagging! +For topic gripper_width: timestamp difference is -222685367 for nearest: 1736304837505904406 - target: 1736304837728589773 at idx 65 +[INFO] [1736304860.422004]: Storing episode... +[print_text-17] killing on exit +[rosbag_record-18] killing on exit +[pub_text-16] killing on exit +[INFO] [1736304861.205630]: Stored episode 6. +[INFO] [1736304861.206840]: Saving video... +[INFO] [1736304861.636119]: Video saved! +Starting episode 6 +[INFO] [1736304861.690580]: resetting environment +[INFO] [1736304861.720416]: cleared current plan +[INFO] [1736304861.729881]: moving to home +[INFO] [1736304865.036837]: reached home +[INFO] [1736304865.048110]: reset action history +[INFO] [1736304867.847255]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304876.131105]: resetting environment +[INFO] [1736304876.161800]: cleared current plan +[INFO] [1736304876.168844]: moving to home +[INFO] [1736304877.177460]: reached home +[INFO] [1736304877.182199]: reset action history +[INFO] [1736304880.031252]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_6_2025-01-07-21-54-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:34475/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2238481] +process[print_text-20]: started with pid [2238482] +process[rosbag_record-21]: started with pid [2238483] +started bagging! +For topic gripper_width: timestamp difference is -101543743 for nearest: 1736304906505900067 - target: 1736304906607443810 at idx 249 +[INFO] [1736304908.876149]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[pub_text-19] killing on exit +[INFO] [1736304909.264807]: Stored episode 7. +[INFO] [1736304909.269282]: Saving video... +[INFO] [1736304909.672487]: Video saved! +Starting episode 7 +[INFO] [1736304909.738489]: resetting environment +[INFO] [1736304909.765723]: cleared current plan +[INFO] [1736304909.769481]: moving to home +[INFO] [1736304913.176738]: reached home +[INFO] [1736304913.187658]: reset action history +[INFO] [1736304916.118587]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304920.127978]: resetting environment +[INFO] [1736304920.173641]: cleared current plan +[INFO] [1736304920.183682]: moving to home +[INFO] [1736304921.190065]: reached home +[INFO] [1736304921.196712]: reset action history +[INFO] [1736304924.053372]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_7_2025-01-07-21-55-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:42157/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2238593] +process[print_text-23]: started with pid [2238617] +process[rosbag_record-24]: started with pid [2238618] +started bagging! +For topic gripper_width: timestamp difference is 116834230 for nearest: 1736304924872700758 - target: 1736304924755866528 at idx 9 +For topic gripper_width: timestamp difference is 116984899 for nearest: 1736304929472568090 - target: 1736304929355583191 at idx 70 +For topic gripper_width: timestamp difference is 100947925 for nearest: 1736304950872646877 - target: 1736304950771698952 at idx 249 +[INFO] [1736304951.052674]: Storing episode... +For topic gripper_width: timestamp difference is 101048389 for nearest: 1736304951072592893 - target: 1736304950971544504 at idx 249 +[print_text-23] killing on exit +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit +[WARN] [1736304952.387765]: Plan exhausted +[WARN] [1736304952.426996]: Plan exhausted +[WARN] [1736304952.468628]: Plan exhausted +[WARN] [1736304952.507027]: Plan exhausted +[INFO] [1736304951.856604]: Stored episode 8. +[INFO] [1736304951.861890]: Saving video... +[INFO] [1736304952.405289]: Video saved! +Starting episode 8 +[INFO] [1736304952.492146]: resetting environment +[INFO] [1736304952.517485]: cleared current plan +[INFO] [1736304952.527679]: moving to home +[INFO] [1736304955.949458]: reached home +[INFO] [1736304955.956584]: reset action history +[INFO] [1736304958.897065]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736304963.552682]: resetting environment +[INFO] [1736304963.607284]: cleared current plan +[INFO] [1736304963.618504]: moving to home +[INFO] [1736304964.629488]: reached home +[INFO] [1736304964.635988]: reset action history +[INFO] [1736304967.542613]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_8_2025-01-07-21-56-07.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:46219/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2238726] +process[print_text-26]: started with pid [2238727] +process[rosbag_record-27]: started with pid [2238751] +started bagging! +[INFO] [1736304994.532381]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1736304995.355280]: Stored episode 9. +[INFO] [1736304995.368001]: Saving video... +[INFO] [1736304995.712702]: Video saved! +Starting episode 9 +[INFO] [1736304995.797722]: resetting environment +[INFO] [1736304995.835926]: cleared current plan +[INFO] [1736304995.838500]: moving to home +[INFO] [1736304999.246628]: reached home +[INFO] [1736304999.252240]: reset action history +[INFO] [1736305002.189193]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305007.949435]: resetting environment +[INFO] [1736305007.969003]: cleared current plan +[INFO] [1736305007.974010]: moving to home +[INFO] [1736305008.980978]: reached home +[INFO] [1736305008.984975]: reset action history +[INFO] [1736305011.877200]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_9_2025-01-07-21-56-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:37083/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2238890] +process[print_text-29]: started with pid [2238891] +process[rosbag_record-30]: started with pid [2238892] +started bagging! +For topic gripper_width: timestamp difference is 100674436 for nearest: 1736305033672592447 - target: 1736305033571918011 at idx 249 +For topic gripper_width: timestamp difference is 101569902 for nearest: 1736305035472594749 - target: 1736305035371024847 at idx 249 +[INFO] [1736305043.796493]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit +[pub_text-28] killing on exit +[INFO] [1736305044.638759]: Stored episode 10. +[INFO] [1736305044.649766]: Saving video... +[INFO] [1736305045.032037]: Video saved! +Starting episode 10 +[INFO] [1736305045.148512]: resetting environment +[INFO] [1736305045.178460]: cleared current plan +[INFO] [1736305045.183850]: moving to home +[WARN] [1736305045.106324]: Plan exhausted +[WARN] [1736305045.152324]: Plan exhausted +[INFO] [1736305048.798535]: reached home +[INFO] [1736305048.808205]: reset action history +[INFO] [1736305051.737838]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305055.641902]: resetting environment +[INFO] [1736305055.677737]: cleared current plan +[INFO] [1736305055.679272]: moving to home +[INFO] [1736305056.700807]: reached home +[INFO] [1736305056.717301]: reset action history +[INFO] [1736305059.680794]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_10_2025-01-07-21-57-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:42179/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2239024] +process[print_text-32]: started with pid [2239025] +process[rosbag_record-33]: started with pid [2239026] +started bagging! +[INFO] [1736305090.964743]: Storing episode... +[print_text-32] killing on exit +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[INFO] [1736305091.830817]: Stored episode 11. +[INFO] [1736305091.840552]: Saving video... +[WARN] [1736305092.306618]: Plan exhausted +[WARN] [1736305092.346302]: Plan exhausted +[WARN] [1736305092.387415]: Plan exhausted +[WARN] [1736305092.441972]: Plan exhausted +[WARN] [1736305092.467100]: Plan exhausted +[WARN] [1736305092.509435]: Plan exhausted +[WARN] [1736305092.546464]: Plan exhausted +[WARN] [1736305092.586302]: Plan exhausted +[WARN] [1736305092.631577]: Plan exhausted +[WARN] [1736305092.667941]: Plan exhausted +[WARN] [1736305092.713227]: Plan exhausted +[WARN] [1736305092.751630]: Plan exhausted +[INFO] [1736305092.578387]: Video saved! +Starting episode 11 +[INFO] [1736305092.677136]: resetting environment +[INFO] [1736305092.773778]: cleared current plan +[INFO] [1736305092.786570]: moving to home +[INFO] [1736305096.119692]: reached home +[INFO] [1736305096.145390]: reset action history +[INFO] [1736305099.137205]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305103.793208]: resetting environment +[INFO] [1736305103.832896]: cleared current plan +[INFO] [1736305103.838618]: moving to home +[INFO] [1736305104.849107]: reached home +[INFO] [1736305104.863519]: reset action history +[INFO] [1736305107.902597]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_11_2025-01-07-21-58-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:43067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2239160] +process[print_text-35]: started with pid [2239161] +process[rosbag_record-36]: started with pid [2239162] +started bagging! +For topic gripper_width: timestamp difference is 100430260 for nearest: 1736305115272570143 - target: 1736305115172139883 at idx 107 +For topic gripper_width: timestamp difference is 101078990 for nearest: 1736305116072600844 - target: 1736305115971521854 at idx 116 +For topic gripper_width: timestamp difference is 102478018 for nearest: 1736305117872570505 - target: 1736305117770092487 at idx 142 +For topic gripper_width: timestamp difference is 103754325 for nearest: 1736305119472569509 - target: 1736305119368815184 at idx 164 +For topic gripper_width: timestamp difference is 104357724 for nearest: 1736305120272554402 - target: 1736305120168196678 at idx 174 +For topic gripper_width: timestamp difference is 104550924 for nearest: 1736305120472592439 - target: 1736305120368041515 at idx 176 +For topic gripper_width: timestamp difference is 105936313 for nearest: 1736305127872568631 - target: 1736305127766632318 at idx 249 +For topic gripper_width: timestamp difference is 106389175 for nearest: 1736305128072607647 - target: 1736305127966218472 at idx 249 +[INFO] [1736305139.087464]: Storing episode... +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit[print_text-35] killing on exit +[INFO] [1736305139.865236]: Stored episode 12. +[INFO] [1736305139.891228]: Saving video... +[WARN] [1736305140.391900]: Plan exhausted +[WARN] [1736305140.427401]: Plan exhausted +[WARN] [1736305140.467909]: Plan exhausted +[WARN] [1736305140.506477]: Plan exhausted +[WARN] [1736305140.547134]: Plan exhausted +[WARN] [1736305140.588208]: Plan exhausted +[WARN] [1736305140.630072]: Plan exhausted +[WARN] [1736305140.673128]: Plan exhausted +[WARN] [1736305140.712002]: Plan exhausted +[WARN] [1736305140.746338]: Plan exhausted +[INFO] [1736305140.530565]: Video saved! +Starting episode 12 +[INFO] [1736305140.667152]: resetting environment +[INFO] [1736305140.751495]: cleared current plan +[INFO] [1736305140.757668]: moving to home +[INFO] [1736305144.067301]: reached home +[INFO] [1736305144.086146]: reset action history +[INFO] [1736305147.172958]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305151.993989]: resetting environment +[INFO] [1736305152.030811]: cleared current plan +[INFO] [1736305152.035578]: moving to home +[INFO] [1736305153.054270]: reached home +[INFO] [1736305153.057586]: reset action history +[INFO] [1736305156.028770]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_12_2025-01-07-21-59-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:38325/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2239295] +process[print_text-38]: started with pid [2239296] +process[rosbag_record-39]: started with pid [2239297] +started bagging! +For topic gripper_width: timestamp difference is 114371476 for nearest: 1736305174172564921 - target: 1736305174058193445 at idx 248 +For topic gripper_width: timestamp difference is 123430899 for nearest: 1736305185172574452 - target: 1736305185049143553 at idx 249 +[INFO] [1736305189.151854]: Storing episode... +[pub_text-37] killing on exit +[rosbag_record-39] killing on exit +[print_text-38] killing on exit +[WARN] [1736305190.509174]: Plan exhausted +[WARN] [1736305190.566019]: Plan exhausted +[WARN] [1736305190.594360]: Plan exhausted +[WARN] [1736305190.628554]: Plan exhausted +[WARN] [1736305190.667300]: Plan exhausted +[WARN] [1736305190.706412]: Plan exhausted +[INFO] [1736305190.026723]: Stored episode 13. +[INFO] [1736305190.060981]: Saving video... +[INFO] [1736305190.541473]: Video saved! +Starting episode 13 +[INFO] [1736305190.698254]: resetting environment +[INFO] [1736305190.733542]: cleared current plan +[INFO] [1736305190.756772]: moving to home +[INFO] [1736305194.380683]: reached home +[INFO] [1736305194.406312]: reset action history +[INFO] [1736305197.534523]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305201.715904]: resetting environment +[INFO] [1736305201.820401]: cleared current plan +[INFO] [1736305201.848129]: moving to home +[INFO] [1736305202.861079]: reached home +[INFO] [1736305202.888431]: reset action history +[INFO] [1736305206.035913]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_13_2025-01-07-22-00-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:45585/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2239429] +process[print_text-41]: started with pid [2239430] +process[rosbag_record-42]: started with pid [2239431] +started bagging! +[INFO] [1736305239.723407]: Storing episode... +[print_text-41] killing on exit +[rosbag_record-42] killing on exit +[pub_text-40] killing on exit +[WARN] [1736305241.069333]: Plan exhausted +[INFO] [1736305240.598220]: Stored episode 14. +[INFO] [1736305240.615555]: Saving video... +[INFO] [1736305240.922941]: Video saved! +Starting episode 14 +[INFO] [1736305241.048642]: resetting environment +[INFO] [1736305241.104746]: cleared current plan +[INFO] [1736305241.128438]: moving to home +[INFO] [1736305244.551247]: reached home +[INFO] [1736305244.570768]: reset action history +[INFO] [1736305247.612719]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736305257.711579]: resetting environment +[INFO] [1736305257.797629]: cleared current plan +[INFO] [1736305257.808214]: moving to home +[INFO] [1736305258.835760]: reached home +[INFO] [1736305258.854972]: reset action history +[INFO] [1736305261.978045]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/episode_rosbags/episode_14_2025-01-07-22-01-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2237375.log +started roslaunch server http://158.130.50.37:37839/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2239565] +process[print_text-44]: started with pid [2239566] +process[rosbag_record-45]: started with pid [2239567] +started bagging! +For topic gripper_width: timestamp difference is -108043944 for nearest: 1736305283605901206 - target: 1736305283713945150 at idx 249 +[INFO] [1736305294.191328]: Storing episode... +[rosbag_record-45] killing on exit +[print_text-44] killing on exit +[pub_text-43] killing on exit +[INFO] [1736305294.940003]: Stored episode 15. +[INFO] [1736305294.953732]: Saving video... +[INFO] [1736305295.522532]: Video saved! +[INFO] [1736305295.702504]: resetting environment +[INFO] [1736305295.764904]: cleared current plan +[INFO] [1736305295.775611]: moving to home +[WARN] [1736305295.513376]: Plan exhausted +[WARN] [1736305295.555498]: Plan exhausted +[WARN] [1736305295.589251]: Plan exhausted +[WARN] [1736305295.631847]: Plan exhausted +[WARN] [1736305295.680236]: Plan exhausted +[WARN] [1736305295.713717]: Plan exhausted +[WARN] [1736305295.746656]: Plan exhausted +[INFO] [1736305299.193411]: reached home +[INFO] [1736305299.212190]: reset action history +[INFO] [1736305302.345410]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..57a746025a5907c4f24aaa7a96ee549e3fbcbcfa --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T02:49:39.378676", + "startedAt": "2025-01-08T02:49:39.039301", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3558.6397500000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4485.477, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.04, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.251, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 637.9640197753906 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..63d0ac9d11f07db3cb058f1cd91824cee8abce1e --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "165af090a1eeabfb388bdb91193faa030692627f90627267d80076485db84af2", "size": 1198949, "path": "media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4"}, "global_step": 1994, "_timestamp": 1736305308.1815999, "_runtime": 729.1352617740631, "_step": 31, "eval/1_eval": {"_type": "video-file", "sha256": "a4095f09212ed4cd67c65f8775628ad42b68590c26b78ab328b3e7265cdce459", "size": 1207724, "path": "media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4"}, "eval/num_success": 1.0, "episode": 85.0, "eval/success_rate": 0.06666667014360428, "eval/success_rate_lb_0.01": 0.0021755192428827286, "eval/success_rate_ub_0.01": 0.3521530032157898, "eval/success_rate_lb_0.025": 0.007666284218430519, "eval/success_rate_ub_0.025": 0.29566386342048645, "eval/success_rate_lb_0.05": 0.016987718641757965, "eval/success_rate_ub_0.05": 0.23827329277992249, "eval/success_rate_lb_0.1": 0.02840900421142578, "eval/success_rate_ub_0.1": 0.18168705701828003, "eval/2_eval": {"_type": "video-file", "sha256": "f1a3689635b40f3325683ba3e3cb26e079f0b0afe9adc75b43296b68b24b8af4", "size": 1216523, "path": "media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "209651fa62317471e2aef900bc59c84ea481b2c3cbcde1a3c9f2109b6fa9d496", "size": 1219522, "path": "media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "83ae6c438be885a5e86e7108b0d37c1e0b43a0186c90c098bd710d9fc3afaf56", "size": 1205814, "path": "media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "5c374bcb2343831d6c96aae9c50249b7f50652c6eecd1c5aa2b3e58eb9e1948c", "size": 1198973, "path": "media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "1fd0a12988dce00dfb925d718a3a1c0a1fe9730fcaed1281e9590f59aaf3cdd5", "size": 1263632, "path": "media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "8ba49b1c68cf3a00e67a25fde51a6f248444542077115544ccbb748a2daa6b56", "size": 1209461, "path": "media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "d2cf0a27b6e7a9a0aea7fd2ff959017bd547f0147819f4f53da4571dd46037a4", "size": 1214532, "path": "media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "eda3ab83a50fa0b2dcdcf5ad8b124e14ebc8cef3e14800250c2afb513a023ef5", "size": 1320353, "path": "media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "cc846afee20f9965cbd4c9c5df83d573e6400ec5460c50cd9b53968e95526f48", "size": 1350850, "path": "media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "bac8f9c871b47a881b7078cf387c6f932e510097420cba7f35847ae6d6af4fc4", "size": 1343960, "path": "media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "81d8d04ea7edc7b1c818d56fc72dd6a7fb05ef251ba2c98faa900c65a799918a", "size": 1372727, "path": "media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "eac1e3a9ea31f7ab5b3ff57bc4ef9121c87cddfe2da60f3b4316d84ac95905db", "size": 1449690, "path": "media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "0dac4d6100436c1334e46d2831ed52c015b258163d6956a4fc34400b4f558a8b", "size": 1395817, "path": "media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "24d8ed5706c93157f17ac94e2e67a6fc480a7911c687cf1b13963642cfafe3f1", "size": 427, "artifact_path": "wandb-client-artifact://t7jyodzbol4lvvg0crkq5qnyrruhgs4b5n53xk0e4lz505azwh9p5lrinx1btbaton79ac5vyihgspbmu2gafir5scww7js9gbwxhj3v4uv7dzden5vp9aoba35prz81/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://whvvc3e4o0whnz1eusqn9wgms6wb29f51e4p61wh7wprto41b896sxxh40lj8cwmxz04dndmphno02q4jcwtlwn3x1h0xsa9wm4mnm7djyipztagacfejs0jeyppvzec:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 35, "total_num_episodes": 85, "total_success_rate": 0.4117647058823529, "total_success_rate_lb_0.01": 0.295654282014708, "total_success_rate_ub_0.01": 0.5383982211274597, "total_success_rate_lb_0.025": 0.3164127469429979, "total_success_rate_ub_0.025": 0.5167854353751943, "total_success_rate_lb_0.05": 0.32573482397703757, "total_success_rate_ub_0.05": 0.5000778660178029, "total_success_rate_lb_0.1": 0.33988951894862907, "total_success_rate_ub_0.1": 0.4838198050888951, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "706adff9374d8bd6f9a23813ba4801c610d71b22260b3f12b5bc8fd93c2b8299", "size": 521, "artifact_path": "wandb-client-artifact://jj2txhvicp06nwl8s6wfpzf14riqkytejab9rh3tx9j8uljgblehwg0fii0esr5b49um05fy4rnr31whg8yqvqdisar9nk4xcs15y91h5qt7yzdpzwcdub6m2u8j4hrh/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zig9w3unna07kzdomzhmocv0dyjy90jim67o3gp6nixo3r0b9thrdcmo47jawlcli5tw2g1257p7ro2jl7oabkiw4m5pmonmio3kvl9amn3m906u4hw2wkxefo04b5ut:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 745}, "eval/total_success_rate": 0.4117647111415863, "eval/total_success_rate_lb_0.01": 0.295654296875, "eval/total_success_rate_ub_0.01": 0.5383982062339783, "eval/total_success_rate_lb_0.025": 0.3164127469062805, "eval/total_success_rate_ub_0.025": 0.5167854428291321, "eval/total_success_rate_lb_0.05": 0.32573482394218445, "eval/total_success_rate_ub_0.05": 0.5000778436660767, "eval/total_success_rate_lb_0.1": 0.3398895263671875, "eval/total_success_rate_ub_0.1": 0.4838198125362396} \ No newline at end of file diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug-internal.log b/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5babf183aeb3be59bed481092c884573266192af --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug-internal.log @@ -0,0 +1,1686 @@ +2025-01-07 21:49:39,046 INFO StreamThr :2237540 [internal.py:wandb_internal():85] W&B internal server running at pid: 2237540, started at: 2025-01-07 21:49:39.046048 +2025-01-07 21:49:39,047 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status +2025-01-07 21:49:39,061 INFO WriterThread:2237540 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb +2025-01-07 21:49:39,062 DEBUG SenderThread:2237540 [sender.py:send():379] send: header +2025-01-07 21:49:39,069 DEBUG SenderThread:2237540 [sender.py:send():379] send: run +2025-01-07 21:49:39,302 INFO SenderThread:2237540 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files +2025-01-07 21:49:39,302 INFO SenderThread:2237540 [sender.py:_start_run_threads():1188] run started: 7u1dchof with start time 1736304579.046338 +2025-01-07 21:49:39,316 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 21:49:39,316 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: check_version +2025-01-07 21:49:39,361 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 21:49:39,366 DEBUG HandlerThread:2237540 [system_info.py:__init__():26] System info init +2025-01-07 21:49:39,366 DEBUG HandlerThread:2237540 [system_info.py:__init__():41] System info init done +2025-01-07 21:49:39,366 INFO HandlerThread:2237540 [system_monitor.py:start():194] Starting system monitor +2025-01-07 21:49:39,366 INFO SystemMonitor:2237540 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 21:49:39,366 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 21:49:39,367 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started disk monitoring +2025-01-07 21:49:39,367 INFO HandlerThread:2237540 [system_monitor.py:probe():214] Collecting system info +2025-01-07 21:49:39,367 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 21:49:39,368 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started memory monitoring +2025-01-07 21:49:39,368 INFO SystemMonitor:2237540 [interfaces.py:start():188] Started network monitoring +2025-01-07 21:49:39,378 DEBUG HandlerThread:2237540 [system_info.py:probe():152] Probing system +2025-01-07 21:49:39,380 DEBUG HandlerThread:2237540 [system_info.py:_probe_git():137] Probing git +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:_probe_git():145] Probing git done +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:probe():200] Probing system done +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T02:49:39.378676', 'startedAt': '2025-01-08T02:49:39.039301', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3558.6397500000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4485.477, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.04, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.251, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 637.9640197753906}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 21:49:39,384 INFO HandlerThread:2237540 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 21:49:39,384 INFO HandlerThread:2237540 [system_monitor.py:probe():227] Publishing system info +2025-01-07 21:49:39,384 DEBUG HandlerThread:2237540 [system_info.py:_save_code():44] Saving code +2025-01-07 21:49:39,386 DEBUG HandlerThread:2237540 [system_info.py:_save_code():67] Saving code done +2025-01-07 21:49:39,386 DEBUG HandlerThread:2237540 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 21:49:39,451 DEBUG HandlerThread:2237540 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 21:49:39,451 INFO HandlerThread:2237540 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 21:49:39,454 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:49:39,454 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 21:49:39,455 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 21:49:39,455 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 21:49:39,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 21:49:39,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:49:39,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: python_packages +2025-01-07 21:49:39,694 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ohqyrws2-wandb-metadata.json +2025-01-07 21:49:39,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:49:39,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:39,697 INFO HandlerThread:2237540 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/tb" + save: true +} +control { + relay_id: "b280c0d6940a4957b38b21bf92a79360" +} +_info { + stream_id: "7u1dchof" +} + +2025-01-07 21:49:39,697 DEBUG HandlerThread:2237540 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 21:49:39,705 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/k05q5dgz-diff.patch +2025-01-07 21:49:39,760 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ukefgs7x-code/FISH/eval_robot.py +2025-01-07 21:49:39,797 DEBUG SenderThread:2237540 [sender.py:send():379] send: telemetry +2025-01-07 21:49:39,798 DEBUG SenderThread:2237540 [sender.py:send():379] send: tbrecord +2025-01-07 21:49:39,798 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:49:39,798 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 with policy live +2025-01-07 21:49:40,017 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lr08xa1p-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:49:40,303 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code +2025-01-07 21:49:40,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH +2025-01-07 21:49:40,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:41,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:42,303 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:42,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:43,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:44,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:44,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:45,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:46,481 DEBUG SenderThread:2237540 [sender.py:send():379] send: config +2025-01-07 21:49:46,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:47,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:48,304 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:48,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:49,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:49,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:50,305 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:50,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:51,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:52,306 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:52,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:53,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:54,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:49:54,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:49:54,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:54,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:49:55,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:56,307 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:56,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:57,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:58,307 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:49:58,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:49:59,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:00,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:00,308 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:00,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:01,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:02,309 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:02,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:03,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:04,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:05,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:05,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:06,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:07,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:08,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:09,693 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:09,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:10,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:10,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:11,311 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 21:50:11,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:12,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:13,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:14,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:15,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:16,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:16,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:17,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:18,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:19,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:20,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:21,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:21,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:22,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:23,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:24,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:24,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:24,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:25,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:26,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:26,773 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:27,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:28,162 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:50:28,162 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 with policy now +2025-01-07 21:50:28,162 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:50:28,163 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:50:28,163 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:50:28,163 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:50:28,315 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:50:28,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos +2025-01-07 21:50:28,461 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lwao7e3p-media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:28,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:28,742 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/r1mxz7xl-media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 21:50:29,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:30,316 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:30,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:31,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:32,621 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:32,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:33,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:34,317 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:34,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:35,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:36,318 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:36,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:37,696 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:37,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:38,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:39,368 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 21:50:39,369 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:50:39,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:39,694 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:39,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:40,319 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:50:40,588 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/pvxw69ed-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:50:40,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:41,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:42,320 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:42,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:43,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:43,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:44,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:45,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:46,321 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:50:46,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:47,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:48,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:48,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:49,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:50,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:51,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:52,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:53,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:53,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:54,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:50:54,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:50:54,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:55,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:56,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:57,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:58,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:50:58,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:50:59,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:00,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:01,705 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:02,326 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:02,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:03,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:03,858 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:04,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:05,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:06,328 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:06,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:07,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:08,328 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:08,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:09,253 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:09,370 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:51:09,694 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:09,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:10,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:11,708 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:12,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:51:12,085 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:51:12,086 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 with policy now +2025-01-07 21:51:12,086 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:51:12,086 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:51:12,086 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:51:12,347 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7w50oiu3-media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,615 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/smxnqzhc-media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 21:51:12,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:13,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:14,330 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:14,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:14,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:15,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:16,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:17,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:18,332 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:18,712 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:19,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:19,712 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:20,333 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:20,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:21,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:22,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:23,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:24,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:24,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:24,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:24,818 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:25,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:26,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:27,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:28,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:29,336 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:51:29,713 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:30,125 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:51:30,126 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:51:30,126 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:30,126 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:51:30,336 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:30,337 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:51:30,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:31,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:32,337 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:32,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:33,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:34,338 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:51:34,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:35,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:35,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:36,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:37,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:38,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:39,370 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:51:39,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:39,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:39,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:40,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:40,816 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:41,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:42,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:43,715 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:44,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:45,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:46,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:46,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:47,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:48,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:49,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:50,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:51,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:52,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:52,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:53,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:54,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:51:54,695 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:51:54,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:55,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:56,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:57,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:57,834 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:51:58,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:51:59,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:00,725 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:01,545 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:52:01,545 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 with policy now +2025-01-07 21:52:01,549 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:52:01,549 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:01,550 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:01,551 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:01,721 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:01,810 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/u6icwa1c-media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:52:02,660 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/bwtpnajq-media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 21:52:02,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:03,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:03,573 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:03,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:04,348 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:04,728 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:05,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:06,349 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:06,728 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:07,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:08,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:08,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:09,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:52:09,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:09,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:09,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:10,350 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:10,731 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:11,736 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:12,734 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:13,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:14,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:14,736 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:15,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:16,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:17,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:18,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:19,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:20,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:20,741 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:21,354 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:52:21,606 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8p4tfv7o-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:52:21,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:21,946 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:21,946 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:21,946 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:22,354 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:22,355 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:22,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:23,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:24,355 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:24,698 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:24,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:24,748 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:25,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:26,356 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:26,495 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:26,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:27,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:28,357 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:28,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:29,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:30,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:31,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:31,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:32,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:33,743 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:34,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:35,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:36,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:37,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:37,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:38,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:39,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:52:39,695 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:39,696 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:39,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:40,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:41,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:42,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:42,811 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:43,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:44,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:45,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:46,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:47,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:48,706 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:48,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:49,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:50,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:51,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:52,748 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:53,606 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:52:53,606 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 with policy now +2025-01-07 21:52:53,609 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:52:53,610 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:52:53,610 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:52:53,610 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:52:53,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:53,870 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/wqvd6pty-media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:52:54,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,366 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:52:54,639 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:54,662 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/c9y1qb0c-media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 21:52:54,703 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:52:54,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:52:54,753 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:55,365 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:55,755 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:56,366 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:56,760 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:57,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:58,367 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:52:58,757 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:52:59,714 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:52:59,756 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:00,752 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:01,753 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:02,368 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:02,755 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:03,758 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:04,758 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:05,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:05,770 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:06,369 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:06,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:07,004 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:07,004 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:07,005 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:07,370 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:07,769 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:08,370 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:08,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:09,371 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:53:09,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:09,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:09,767 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:10,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:11,506 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:11,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:12,371 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:12,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:13,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:14,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:15,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:16,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:16,761 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:17,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:18,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:19,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:20,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:21,707 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:21,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:22,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:23,762 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:24,699 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:24,699 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:24,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:25,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:26,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:26,837 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:27,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:28,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:29,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:30,763 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:31,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:32,708 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:32,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:33,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:34,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:35,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:36,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:37,768 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:38,004 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:38,384 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:38,692 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:53:38,692 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 with policy now +2025-01-07 21:53:38,697 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:53:38,697 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:38,698 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:38,698 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:38,771 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:38,949 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/k3ukfryr-media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,372 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:53:39,675 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7c94hrg0-media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 21:53:39,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:39,777 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:40,385 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:40,774 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:41,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:42,782 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:43,077 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:43,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:44,386 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:44,788 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:45,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:46,387 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:46,784 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:47,783 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:48,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:48,870 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:49,788 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:50,056 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:53:50,057 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:53:50,057 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:50,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:53:50,648 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8k5znsay-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:53:50,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:51,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:52,389 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:52,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:53,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:54,390 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:54,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:53:54,700 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:53:54,781 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:53:54,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:55,789 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:56,391 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:53:56,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:57,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:58,790 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:53:59,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:00,392 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:00,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:00,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:01,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:02,791 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:03,792 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:04,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:05,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:05,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:06,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:07,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:08,793 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:09,372 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:54:09,700 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:09,700 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:09,794 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:10,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:10,850 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:11,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:12,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:13,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:14,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:15,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:16,711 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:16,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:17,797 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:18,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:19,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:20,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:21,677 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:54:21,677 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 with policy now +2025-01-07 21:54:21,683 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:54:21,684 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:54:21,684 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:54:21,684 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:54:21,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:21,807 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:21,936 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/2l8065zn-media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,404 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:54:22,686 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/crb62mbz-media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 21:54:22,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:23,805 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:24,405 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:24,711 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:24,711 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:24,803 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:25,812 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:26,405 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:26,806 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:27,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:27,807 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:28,808 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:29,809 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:30,407 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:30,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:31,815 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:32,716 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:32,816 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:33,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:34,818 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:35,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:36,112 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:54:36,112 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:54:36,113 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:54:36,409 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:54:36,409 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:54:36,828 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:37,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:38,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:38,410 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:38,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:39,373 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:54:39,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:39,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:40,825 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:41,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:42,411 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:42,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:43,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:44,174 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:44,412 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:54:44,820 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:45,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:46,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:47,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:48,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:49,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:49,821 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:50,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:51,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:52,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:53,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:54,701 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:54:54,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:54:54,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:54,827 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:54:55,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:56,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:57,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:58,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:54:59,822 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:00,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:00,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:01,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:02,823 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:03,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:04,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:05,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:06,717 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:06,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:07,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:08,421 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:08,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:09,373 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:55:09,722 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:09,722 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:09,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:55:09,824 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:55:09,824 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 with policy now +2025-01-07 21:55:09,824 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:09,825 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:09,825 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:09,825 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:10,083 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/r120ktgi-media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,421 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:55:10,699 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/tbzwhest-media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 21:55:10,833 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:11,826 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:11,828 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:12,422 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:12,841 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:13,849 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:14,423 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:14,849 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:15,852 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:16,851 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:17,161 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:17,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:18,425 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:18,850 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:19,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:20,167 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:20,168 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:20,168 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:20,426 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:55:20,426 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:20,682 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/lhd51kd1-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:55:20,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:21,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:22,233 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:22,427 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:22,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:23,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:24,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:24,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:24,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:25,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:26,429 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:26,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:27,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:27,922 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:28,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:29,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:30,429 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:30,853 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:31,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:32,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:33,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:33,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:34,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:35,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:36,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:37,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:38,720 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:38,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:39,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:55:39,702 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:39,702 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:39,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:40,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:41,861 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:42,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:43,803 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:43,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:44,855 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:45,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:46,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:47,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:48,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:49,721 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:49,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:50,857 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:51,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:52,437 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:52,478 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:55:52,478 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 with policy now +2025-01-07 21:55:52,482 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:55:52,483 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:55:52,483 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:55:52,484 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:55:52,735 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/tz4dzcsu-media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:52,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:53,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:55:53,739 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/jrod8liv-media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 21:55:53,869 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:54,438 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:54,709 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:55:54,709 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:55:54,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:55:54,866 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:55,874 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:56,888 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:57,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:58,440 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:55:58,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:55:59,878 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:00,440 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:00,738 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:00,878 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:01,879 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:02,877 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:03,881 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:04,224 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:04,224 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:04,225 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:04,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:04,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:04,443 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:04,881 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:05,900 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:06,442 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:06,726 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:06,891 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:07,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:08,443 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:08,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:09,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:56:09,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:09,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:09,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:10,444 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:10,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:11,842 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:11,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:12,882 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:13,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:14,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:15,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:16,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:17,725 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:17,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:18,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:19,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:20,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:21,883 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:22,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:23,723 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:23,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:24,704 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:24,704 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:24,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:25,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:26,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:27,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:28,796 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:28,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:29,884 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:30,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:31,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:32,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:33,885 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:34,532 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:34,901 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:35,771 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:56:35,772 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 with policy now +2025-01-07 21:56:35,786 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:56:35,787 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:35,787 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:35,787 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:35,902 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:36,028 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/8oizt1zu-media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:56:36,777 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/p0lzpr9w-media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 21:56:36,906 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:37,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:38,454 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:38,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:39,374 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:56:39,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:39,730 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:39,795 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:39,913 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:40,455 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:40,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:41,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:42,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:43,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:44,456 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:44,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:45,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:45,919 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:46,918 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:47,927 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:48,279 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:56:48,279 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:56:48,280 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:56:48,458 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:48,458 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:56:48,714 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/2won4f61-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:56:48,939 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:49,942 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:50,459 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:50,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:50,943 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:51,941 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:52,955 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:53,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:54,460 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:54,734 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:56:54,734 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:56:54,951 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:55,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:56:55,961 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:56,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:57,944 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:58,462 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:56:58,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:56:59,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:00,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:01,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:01,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:02,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:03,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:04,945 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:05,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:06,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:07,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:07,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:08,946 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:09,375 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:57:09,718 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:09,718 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:09,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:10,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:11,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:12,786 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:12,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:13,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:14,468 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:14,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:15,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:16,469 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:16,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:17,947 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:18,521 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:18,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:19,950 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:20,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:21,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:22,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:23,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:23,950 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:24,472 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:24,740 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:24,740 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:24,953 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:25,132 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:57:25,132 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 with policy now +2025-01-07 21:57:25,138 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:57:25,138 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:57:25,139 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:57:25,139 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:57:25,394 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/rs8mej0e-media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:57:25,759 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/3gxbrza1-media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 21:57:25,964 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:26,473 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:26,970 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:27,959 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:28,474 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:28,799 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:28,997 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:29,983 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:30,474 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:30,991 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:31,979 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:32,476 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:32,992 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:33,993 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:34,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:34,984 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:36,001 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:36,333 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:57:36,333 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:57:36,334 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:57:36,477 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:57:36,477 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:36,478 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:57:37,017 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:38,009 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:38,478 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:39,012 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:39,375 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:57:39,729 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:39,729 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:39,874 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:40,018 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:40,479 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:41,036 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:42,011 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:43,011 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:44,017 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:44,485 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:45,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:45,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:46,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:46,486 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:57:47,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:48,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:49,013 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:50,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:51,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:51,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:52,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:53,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:54,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:54,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:57:54,719 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:57:55,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:56,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:56,864 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:57:57,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:58,014 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:57:59,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:00,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:01,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:02,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:02,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:03,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:04,015 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:05,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:06,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:07,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:07,742 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:08,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:09,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:09,376 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:58:09,719 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:09,719 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:10,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:11,016 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:12,044 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:12,495 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:12,661 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:58:12,662 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 with policy now +2025-01-07 21:58:12,662 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:58:12,662 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:58:12,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:58:12,663 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:58:12,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:12,913 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/yv4ldzu5-media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:13,051 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:13,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:58:13,796 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/7o9ep2ct-media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 21:58:14,051 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:14,496 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:15,072 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:16,065 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:17,073 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:18,071 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:18,173 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:18,498 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:19,072 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:20,081 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:20,499 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:21,109 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:22,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:23,076 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:23,710 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:24,102 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:24,394 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:58:24,394 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:58:24,394 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:58:24,500 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:58:24,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:24,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:58:24,730 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:24,730 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:24,763 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/sdfhlm89-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:58:25,104 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:26,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:26,501 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:27,087 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:28,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:28,502 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:28,949 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:29,097 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:30,101 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:30,502 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:31,094 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:32,105 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:32,504 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:33,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:34,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:34,247 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:34,505 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:35,082 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:36,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:36,506 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:37,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:38,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:38,506 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:39,083 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:39,376 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:58:39,377 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:39,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:39,724 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:40,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:40,507 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:41,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:42,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:42,508 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:43,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:44,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:44,488 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:45,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:46,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:47,084 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:48,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:48,512 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:49,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:49,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:50,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:50,513 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:58:51,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:52,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:53,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:54,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:54,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:58:54,724 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:58:54,834 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:58:55,085 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:56,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:57,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:58,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:59,086 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:58:59,877 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:00,103 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:00,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:00,643 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:59:00,643 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 with policy now +2025-01-07 21:59:00,652 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:59:00,653 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:00,653 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:00,653 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:00,902 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/h2bw89nv-media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:01,091 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:01,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:59:01,828 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/l7gc7soe-media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 21:59:02,096 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:02,518 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:03,104 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:04,129 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:05,122 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:05,773 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:06,114 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:06,520 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:07,123 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:08,132 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:08,521 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:09,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:09,377 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:59:09,737 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:09,738 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:10,127 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:10,798 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:11,141 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:12,123 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:12,464 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:12,464 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:12,464 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:12,522 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 21:59:12,522 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:12,523 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:13,125 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:14,137 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:14,523 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:15,133 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:16,031 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:16,132 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:17,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:18,162 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:18,525 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:19,150 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:20,149 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:21,176 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:21,777 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:22,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:23,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:24,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:24,527 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:24,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:24,725 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:25,135 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:26,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:26,847 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:27,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:28,134 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:29,135 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:30,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:31,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:32,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:32,766 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:33,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:34,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:35,142 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:36,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:36,531 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:37,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:37,766 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:38,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:39,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:39,377 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 21:59:39,724 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:39,725 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:40,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:41,143 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:42,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:42,854 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:43,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:44,147 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:45,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:46,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:46,539 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:47,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:48,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:48,225 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:49,148 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:50,171 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:50,541 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:50,665 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 21:59:50,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 with policy now +2025-01-07 21:59:50,669 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 21:59:50,669 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 21:59:50,670 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 21:59:50,670 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 21:59:50,930 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/67y65hit-media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:51,165 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:51,541 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 21:59:51,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:51,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 21:59:51,874 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/4edgdnp9-media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 21:59:52,179 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:52,542 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:53,198 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:53,764 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 21:59:54,197 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:54,746 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 21:59:54,746 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 21:59:55,215 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:56,199 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:56,543 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:57,225 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:58,224 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:58,544 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 21:59:59,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 21:59:59,581 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:00,235 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:01,213 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:02,217 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:02,520 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:02,520 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:02,521 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:02,546 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:00:03,215 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:04,201 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:04,547 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:04,814 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:05,244 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:06,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:07,214 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:08,218 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:08,548 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:09,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:09,378 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:00:09,739 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:09,739 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:09,894 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:10,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:11,233 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:12,232 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:13,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:14,220 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:14,551 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:15,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:15,799 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:16,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:17,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:18,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:19,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:20,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:20,801 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:21,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:22,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:23,221 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:24,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:24,732 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:24,732 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:25,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:25,863 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:26,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:27,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:28,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:29,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:30,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:31,222 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:31,800 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:32,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:33,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:34,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:35,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:36,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:36,800 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:37,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:38,224 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:39,223 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:39,378 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:00:39,765 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:39,765 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:40,245 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:40,561 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:41,022 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:00:41,023 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 with policy now +2025-01-07 22:00:41,023 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:00:41,023 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:41,023 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:41,024 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:41,231 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:41,279 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/9wqjt18f-media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:41,561 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:00:41,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:41,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 22:00:41,850 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ta0ravul-media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:00:42,131 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:42,238 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:42,562 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:43,245 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:44,238 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:45,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:46,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:46,563 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:47,264 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:47,628 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:48,265 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:48,564 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:49,290 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:50,286 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:51,280 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:52,294 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:52,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:53,283 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:54,281 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:54,749 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:00:54,749 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:00:55,304 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:56,291 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:57,290 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:58,283 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:58,568 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:58,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:00:58,595 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:00:58,595 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:00:58,595 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:00:58,595 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:00:58,823 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/3zwgkeg1-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:00:59,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:00:59,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:00,310 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:00,569 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:01,320 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:02,293 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:02,570 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:03,295 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:04,053 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:04,304 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:05,308 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:06,315 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:07,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:08,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:08,573 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:09,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:09,379 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:01:09,379 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:09,744 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:09,745 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:10,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:11,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:12,300 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:13,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:14,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:14,809 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:15,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:17,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:18,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:19,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:19,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:20,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:21,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:22,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:23,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:24,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:24,579 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:24,745 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:24,745 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:24,898 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:25,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:26,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:27,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:28,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:29,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:30,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:30,810 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:31,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:32,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:33,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:34,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:35,307 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:35,672 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:35,672 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 with policy now +2025-01-07 22:01:35,682 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:35,682 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:35,682 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:35,682 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:35,939 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/d31nmdix-media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:36,357 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:36,584 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval +2025-01-07 22:01:36,776 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:36,893 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/ib39peo4-media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:01:37,348 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:38,342 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:38,585 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:39,346 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:39,379 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:01:39,751 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:39,751 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:40,361 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:40,585 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:41,388 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:41,893 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:42,409 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:43,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:44,350 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:44,587 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:45,357 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:46,351 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:46,588 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:01:46,588 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:46,652 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:46,653 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:46,653 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:47,362 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:47,488 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:01:47,488 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:01:47,591 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:47,824 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:47,959 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxqm9j9pt +2025-01-07 22:01:48,268 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:01:48,277 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:01:48,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:48,399 INFO SenderThread:2237540 [sender.py:send_request_log_artifact():1525] logged artifact run-7u1dchof-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjE3Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg3ODU0NA==', 'latestArtifact': None}} +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:48,400 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json with policy now +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:48,400 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,401 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:48,401 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,401 DEBUG SenderThread:2237540 [sender.py:send():379] send: telemetry +2025-01-07 22:01:48,401 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,402 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,402 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,402 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,402 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,404 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,404 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,405 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,405 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,405 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,405 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,406 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,406 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,406 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,406 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,407 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,407 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,407 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,408 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,408 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,408 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:48,409 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:48,409 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:01:48,589 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/sft7io99-media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,592 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table +2025-01-07 22:01:48,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval +2025-01-07 22:01:48,836 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/1mfgcnas-media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:48,843 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/gbfljdtf-media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:01:48,854 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxm4ae1ed +2025-01-07 22:01:49,273 INFO SenderThread:2237540 [sender.py:send_request_log_artifact():1525] logged artifact run-7u1dchof-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjE5NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg3ODU0OA==', 'latestArtifact': None}} +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send():379] send: files +2025-01-07 22:01:49,273 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json with policy now +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:01:49,273 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:01:49,274 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:01:49,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:49,533 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/gcn6k838-media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:01:49,593 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:01:50,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:50,594 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:01:51,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:52,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:53,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:53,835 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:54,352 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:54,747 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:01:54,747 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:01:55,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:56,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:57,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:58,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:01:58,856 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:01:59,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:00,353 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:01,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:02,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:03,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:04,354 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:04,371 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:04,413 DEBUG SenderThread:2237540 [sender.py:send():379] send: exit +2025-01-07 22:02:04,413 INFO SenderThread:2237540 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_exit():588] handling runtime: 745 +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_exit():594] send defer +2025-01-07 22:02:04,414 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,414 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 22:02:04,414 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,414 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 1 +2025-01-07 22:02:04,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 22:02:04,415 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 22:02:04,415 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 2 +2025-01-07 22:02:04,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 22:02:04,415 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 22:02:04,415 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 22:02:04,415 DEBUG SystemMonitor:2237540 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 22:02:04,416 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 22:02:04,469 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 22:02:04,469 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 22:02:04,470 INFO HandlerThread:2237540 [interfaces.py:finish():200] Joined network monitor +2025-01-07 22:02:04,470 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,470 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 22:02:04,470 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 3 +2025-01-07 22:02:04,470 DEBUG SenderThread:2237540 [sender.py:send():379] send: stats +2025-01-07 22:02:04,470 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,470 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 22:02:04,471 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:04,471 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 22:02:04,471 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 4 +2025-01-07 22:02:04,471 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:04,471 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 22:02:04,600 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:06,601 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:09,602 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:10,661 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 22:02:10,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,662 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 22:02:10,662 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 5 +2025-01-07 22:02:10,662 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:10,663 DEBUG SenderThread:2237540 [sender.py:send():379] send: history +2025-01-07 22:02:10,663 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:02:10,663 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,663 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:10,665 DEBUG SenderThread:2237540 [sender.py:send():379] send: summary +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:02:10,665 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 22:02:10,665 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 6 +2025-01-07 22:02:10,665 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,666 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 22:02:10,666 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 7 +2025-01-07 22:02:10,666 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:02:10,666 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:10,666 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 22:02:10,666 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:10,666 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 22:02:11,415 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:11,603 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:13,835 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 8 +2025-01-07 22:02:13,835 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:13,836 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:13,836 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 22:02:13,836 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:13,836 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 22:02:13,836 INFO SenderThread:2237540 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 22:02:13,839 INFO SenderThread:2237540 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 22:02:13,840 INFO SenderThread:2237540 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 22:02:13,859 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 9 +2025-01-07 22:02:13,859 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:13,859 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 22:02:13,859 DEBUG SenderThread:2237540 [sender.py:send():379] send: artifact +2025-01-07 22:02:14,416 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:14,541 INFO wandb-upload_1:2237540 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmph0mz9co_ +2025-01-07 22:02:14,605 INFO Thread-12 :2237540 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:14,670 INFO wandb-upload_0:2237540 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpzwxnw11b +2025-01-07 22:02:14,691 INFO wandb-upload_2:2237540 [upload_job.py:push():88] Uploaded file /tmp/tmpods0y954/wandb-job.json +2025-01-07 22:02:15,296 INFO SenderThread:2237540 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjUyMjY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 22:02:15,297 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:15,297 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 22:02:15,297 INFO SenderThread:2237540 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml config.yaml +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json wandb-summary.json +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt requirements.txt +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-metadata.json wandb-metadata.json +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log output.log +2025-01-07 22:02:15,606 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/diff.patch diff.patch +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 media/videos/eval/2_eval_2_f1a3689635b40f332568.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 media/videos/eval/6_eval_6_1fd0a12988dce00dfb92.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 media/videos/eval/12_eval_12_81d8d04ea7edc7b1c818.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 media/videos/eval/10_eval_10_cc846afee20f9965cbd4.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 media/videos/eval/3_eval_3_209651fa62317471e2ae.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 media/videos/eval/9_eval_9_eda3ab83a50fa0b2dcdc.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 media/videos/eval/4_eval_4_83ae6c438be885a5e86e.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 media/videos/eval/11_eval_11_bac8f9c871b47a881b70.mp4 +2025-01-07 22:02:15,610 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 media/videos/eval/13_eval_13_eac1e3a9ea31f7ab5b3f.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 media/videos/eval/0_eval_0_165af090a1eeabfb388b.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 media/videos/eval/8_eval_8_d2cf0a27b6e7a9a0aea7.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 media/videos/eval/7_eval_7_8ba49b1c68cf3a00e67a.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 media/videos/eval/1_eval_1_a4095f09212ed4cd67c6.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 media/videos/eval/14_eval_14_0dac4d6100436c1334e4.mp4 +2025-01-07 22:02:15,611 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 media/videos/eval/5_eval_5_5c374bcb2343831d6c96.mp4 +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json media/table/eval/success_rate_ci_15_24d8ed5706c93157f17a.table.json +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json media/table/eval/total_success_rate_ci_16_706adff9374d8bd6f9a2.table.json +2025-01-07 22:02:15,617 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 10 +2025-01-07 22:02:15,617 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:15,617 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:15,618 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 22:02:15,618 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:15,618 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 22:02:15,618 INFO SenderThread:2237540 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:02:15,790 INFO wandb-upload_0:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/wandb-summary.json +2025-01-07 22:02:15,796 INFO wandb-upload_1:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/config.yaml +2025-01-07 22:02:15,839 INFO wandb-upload_3:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/requirements.txt +2025-01-07 22:02:15,877 INFO wandb-upload_2:2237540 [upload_job.py:push():130] Uploaded file /tmp/tmp07wha5auwandb/egceoa3o-events.out.tfevents.1736304579.leonmkim-ROG-Strix-G15CS-G15CS.2237375.0 +2025-01-07 22:02:15,887 INFO wandb-upload_4:2237540 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/files/output.log +2025-01-07 22:02:16,087 INFO Thread-11 (_thread_body):2237540 [sender.py:transition_state():614] send defer: 11 +2025-01-07 22:02:16,087 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,087 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 22:02:16,087 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,087 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 22:02:16,088 INFO SenderThread:2237540 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 22:02:16,090 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 12 +2025-01-07 22:02:16,090 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,090 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 22:02:16,091 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,091 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 22:02:16,091 INFO SenderThread:2237540 [file_stream.py:finish():601] file stream finish called +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [file_stream.py:finish():605] file stream finish is done +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 13 +2025-01-07 22:02:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,301 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 22:02:16,301 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 22:02:16,301 INFO SenderThread:2237540 [sender.py:transition_state():614] send defer: 14 +2025-01-07 22:02:16,301 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:02:16,301 INFO HandlerThread:2237540 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send():379] send: final +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send():379] send: footer +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: defer +2025-01-07 22:02:16,302 INFO SenderThread:2237540 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 22:02:16,302 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:16,302 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:02:16,303 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: server_info +2025-01-07 22:02:16,303 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 22:02:16,305 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 22:02:16,305 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:02:16,363 DEBUG SenderThread:2237540 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:02:16,363 INFO MainThread:2237540 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 22:02:16,364 INFO MainThread:2237540 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 22:02:16,364 INFO MainThread:2237540 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 22:02:16,364 DEBUG HandlerThread:2237540 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 22:02:16,364 INFO HandlerThread:2237540 [handler.py:finish():882] shutting down handler +2025-01-07 22:02:17,303 INFO WriterThread:2237540 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [sender.py:finish():1615] shutting down sender +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:02:17,363 INFO SenderThread:2237540 [file_pusher.py:join():175] waiting for file pusher diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug.log b/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b9da1526956ab4525ef5eb9b8cf86a181f8c9fe6 --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Configure stats pid to 2237375 +2025-01-07 21:49:39,040 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/settings +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug.log +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/wandb/run-20250107_214939-7u1dchof/logs/debug-internal.log +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():569] calling init triggers +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 15, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof'} +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():619] starting backend +2025-01-07 21:49:39,041 INFO MainThread:2237375 [wandb_init.py:init():623] setting up manager +2025-01-07 21:49:39,045 INFO MainThread:2237375 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 21:49:39,046 INFO MainThread:2237375 [wandb_init.py:init():631] backend started and connected +2025-01-07 21:49:39,060 INFO MainThread:2237375 [wandb_init.py:init():720] updated telemetry +2025-01-07 21:49:39,067 INFO MainThread:2237375 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 21:49:39,316 INFO MainThread:2237375 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 21:49:39,354 INFO MainThread:2237375 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 21:49:39,355 INFO MainThread:2237375 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 21:49:39,694 INFO MainThread:2237375 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 21:49:39,696 INFO MainThread:2237375 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 21:49:39,697 INFO MainThread:2237375 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/7u1dchof/tb, True +2025-01-07 21:49:46,481 INFO MainThread:2237375 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 22:02:17,365 WARNING MsgRouterThr:2237375 [router.py:message_loop():77] message_loop has been closed diff --git a/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb b/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb new file mode 100644 index 0000000000000000000000000000000000000000..36dd88877e986a1bb8274b77c629d77ec57cffdf --- /dev/null +++ b/7u1dchof/wandb/run-20250107_214939-7u1dchof/run-7u1dchof.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ab31a0965807405ab58ffeefdb098d63dc6b370316816e585686eb5dac8161 +size 181871 diff --git a/8ycpgp3n/.hydra/config.yaml b/8ycpgp3n/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/8ycpgp3n/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/8ycpgp3n/.hydra/hydra.yaml b/8ycpgp3n/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dd7c9a6a7b8b9847db0165eeaa80db4966b3557 --- /dev/null +++ b/8ycpgp3n/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/8ycpgp3n/.hydra/overrides.yaml b/8ycpgp3n/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/8ycpgp3n/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/8ycpgp3n/episode_rosbags/aligned_depth_to_color_K.npy b/8ycpgp3n/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8ycpgp3n/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8ycpgp3n/episode_rosbags/cam_tf_world.npy b/8ycpgp3n/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/8ycpgp3n/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/8ycpgp3n/episode_rosbags/color_K.npy b/8ycpgp3n/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/8ycpgp3n/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/8ycpgp3n/episode_rosbags/depth_K.npy b/8ycpgp3n/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/8ycpgp3n/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/8ycpgp3n/eval_robot.log b/8ycpgp3n/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..1389b89a042509b91534da06e915b95939013526 --- /dev/null +++ b/8ycpgp3n/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:07:36,025][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:07:36,028][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/8ycpgp3n/eval_video/0_eval.mp4 b/8ycpgp3n/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e2c3d9d3000541613cf04aad14e0bf73d098263a --- /dev/null +++ b/8ycpgp3n/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b356ff1064e631440b303c289ad5129b415f5b66f6e26b121c73e79ab327791 +size 1108466 diff --git a/8ycpgp3n/eval_video/1_eval.mp4 b/8ycpgp3n/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab01dd6abf3848cef5f383825ef41461bede7965 --- /dev/null +++ b/8ycpgp3n/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2cbfc725296a7e599308bb92b98fc4f7503e9925f167359530a11b999bc2ac +size 1190976 diff --git a/8ycpgp3n/eval_video/2_eval.mp4 b/8ycpgp3n/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca9953f822c7c421df50b06d4a62d58099e1e3ea --- /dev/null +++ b/8ycpgp3n/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff803fd511eb33dc17ee397bacd4f0752824130ab07d7f6040f228079992066e +size 1202106 diff --git a/8ycpgp3n/eval_video/3_eval.mp4 b/8ycpgp3n/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36349892bddf4e74c620193d4812ccec4b9ed861 --- /dev/null +++ b/8ycpgp3n/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:509210f2f4eb425aa9519dd8da692096f5f7b5e8b5cbfa9aa09fef0f4a71f5e8 +size 1236550 diff --git a/8ycpgp3n/eval_video/4_eval.mp4 b/8ycpgp3n/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c038c4f7dd75826168074979044132fc2b36cd6 --- /dev/null +++ b/8ycpgp3n/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:195f521b8e7cf2a9579a5250a3c557c81dd59fd9cc49166add462c0498d56180 +size 1143381 diff --git a/8ycpgp3n/tb/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 b/8ycpgp3n/tb/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 new file mode 100644 index 0000000000000000000000000000000000000000..14e23705f97e0db1e5676343a9bf1fc347e1bf1e --- /dev/null +++ b/8ycpgp3n/tb/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ba176a57569ece8b22b4b03655744c004a8170bb71ec36390bbd7e28295ca3 +size 6539 diff --git a/8ycpgp3n/wandb/debug-internal.log b/8ycpgp3n/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f9535781ac7a77a78ebd574163881fc0f9275b2 --- /dev/null +++ b/8ycpgp3n/wandb/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-06 17:07:42,087 INFO StreamThr :2085739 [internal.py:wandb_internal():85] W&B internal server running at pid: 2085739, started at: 2025-01-06 17:07:42.087026 +2025-01-06 17:07:42,088 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:07:42,099 INFO WriterThread:2085739 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb +2025-01-06 17:07:42,099 DEBUG SenderThread:2085739 [sender.py:send():379] send: header +2025-01-06 17:07:42,108 DEBUG SenderThread:2085739 [sender.py:send():379] send: run +2025-01-06 17:07:42,367 INFO SenderThread:2085739 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files +2025-01-06 17:07:42,367 INFO SenderThread:2085739 [sender.py:_start_run_threads():1188] run started: 8ycpgp3n with start time 1736201262.087823 +2025-01-06 17:07:42,383 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:07:42,384 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:07:42,456 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:07:42,461 DEBUG HandlerThread:2085739 [system_info.py:__init__():26] System info init +2025-01-06 17:07:42,461 DEBUG HandlerThread:2085739 [system_info.py:__init__():41] System info init done +2025-01-06 17:07:42,461 INFO HandlerThread:2085739 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:07:42,461 INFO SystemMonitor:2085739 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:07:42,461 INFO HandlerThread:2085739 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:07:42,462 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:07:42,463 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:07:42,463 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:07:42,464 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:07:42,464 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:07:42,474 DEBUG HandlerThread:2085739 [system_info.py:probe():152] Probing system +2025-01-06 17:07:42,476 DEBUG HandlerThread:2085739 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:probe():200] Probing system done +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:07:42.474816', 'startedAt': '2025-01-06T22:07:42.079437', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.0075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.117, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.969, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 314.2787742614746}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:07:42,481 INFO HandlerThread:2085739 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:07:42,481 INFO HandlerThread:2085739 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:_save_code():44] Saving code +2025-01-06 17:07:42,482 DEBUG HandlerThread:2085739 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:07:42,482 DEBUG HandlerThread:2085739 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:07:42,544 DEBUG HandlerThread:2085739 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:07:42,544 INFO HandlerThread:2085739 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:07:42,547 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:07:42,547 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:07:42,548 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:07:42,548 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:07:42,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:07:42,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:07:42,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:07:42,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:07:42,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:42,792 INFO HandlerThread:2085739 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/tb" + save: true +} +control { + relay_id: "f007897853a543cc8f6f92970a65dbe3" +} +_info { + stream_id: "8ycpgp3n" +} + +2025-01-06 17:07:42,792 DEBUG HandlerThread:2085739 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:07:42,799 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/hgz8a79o-wandb-metadata.json +2025-01-06 17:07:42,799 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/m8dd2rbx-diff.patch +2025-01-06 17:07:42,848 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/mplv54em-code/FISH/eval_robot.py +2025-01-06 17:07:42,877 DEBUG SenderThread:2085739 [sender.py:send():379] send: telemetry +2025-01-06 17:07:42,877 DEBUG SenderThread:2085739 [sender.py:send():379] send: tbrecord +2025-01-06 17:07:42,878 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:07:42,878 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 with policy live +2025-01-06 17:07:43,070 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/bikec69p-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code +2025-01-06 17:07:43,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:44,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:45,368 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:45,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:46,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:47,332 DEBUG SenderThread:2085739 [sender.py:send():379] send: config +2025-01-06 17:07:47,332 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:47,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:48,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:49,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:49,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:50,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:51,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:51,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:52,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:52,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:53,371 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:53,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:54,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:55,371 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:55,813 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:56,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:57,372 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:57,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:07:57,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:07:57,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:57,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:58,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:59,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:00,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:01,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:02,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:03,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:03,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:04,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:05,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:06,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:07,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:08,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:08,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:09,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:10,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:11,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:12,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:12,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:12,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:13,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:13,939 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:14,376 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:08:14,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:15,377 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:15,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:16,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:17,377 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:17,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:18,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:19,326 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:19,381 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:19,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:20,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:21,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:22,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:23,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:24,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:24,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:25,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:26,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:27,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:27,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:27,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:28,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:29,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:29,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:30,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:31,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:32,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:33,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:34,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:35,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:35,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:36,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:37,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:38,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:39,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:40,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:40,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:41,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:42,465 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:08:42,465 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:08:42,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:42,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:42,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:43,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:44,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:45,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:45,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:46,004 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:46,049 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:08:46,050 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 with policy now +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:08:46,051 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:08:46,310 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/oiocex7m-media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos +2025-01-06 17:08:46,665 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/igx7hdy1-media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:47,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:47,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:48,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:49,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:50,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:51,390 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:51,669 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:51,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:52,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:53,390 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:53,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:54,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:55,391 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:08:55,646 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/s92ufxho-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:08:55,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:56,677 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:56,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:57,392 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:57,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:57,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:57,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:58,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:59,393 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:59,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:00,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:01,394 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:01,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:02,589 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:02,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:03,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:04,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:05,395 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:05,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:06,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:07,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:07,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:08,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:09,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:10,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:11,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:12,466 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:09:12,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:12,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:12,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:12,935 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:13,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:14,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:15,398 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:15,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:16,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:17,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:18,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:18,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:19,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:20,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:21,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:22,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:23,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:24,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:24,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:25,823 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:26,627 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:09:26,627 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 with policy now +2025-01-06 17:09:26,632 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:09:26,633 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:09:26,633 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:09:26,634 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:09:26,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:26,894 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/x1fr96t6-media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,402 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:09:27,688 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/yjfmt6dg-media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,799 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:27,799 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:27,841 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:28,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:29,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:29,830 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:29,930 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:30,831 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:31,404 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:31,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:32,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:33,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:34,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:35,405 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:35,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:35,885 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:36,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:37,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:38,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:39,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:40,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:41,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:41,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:42,408 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:09:42,466 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:09:42,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:42,791 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:42,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:43,217 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:09:43,218 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:09:43,218 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:09:43,408 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:43,409 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:09:43,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:44,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:45,409 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:45,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:46,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:47,410 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:47,428 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:47,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:48,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:49,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:50,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:51,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:52,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:52,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:53,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:54,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:55,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:56,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:57,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:57,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:57,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:58,220 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:58,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:59,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:00,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:01,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:02,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:03,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:03,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:04,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:05,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:06,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:07,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:08,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:09,796 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:09,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:10,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:11,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:12,467 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:10:12,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:12,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:12,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:13,842 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:14,633 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:10:14,633 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 with policy now +2025-01-06 17:10:14,636 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:10:14,637 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:10:14,637 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:10:14,637 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:10:14,840 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:14,928 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/6tlyz576-media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,419 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:10:15,659 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:15,714 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/e2j46bsj-media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,841 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:16,842 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:17,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:17,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:18,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:19,421 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:19,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:20,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:20,849 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:21,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:22,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:23,422 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:23,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:24,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:25,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:26,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:26,852 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:27,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:27,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:27,854 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:28,853 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:29,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:30,859 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:31,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:31,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:32,428 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:10:32,678 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/g35elcwn-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:10:32,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:33,051 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:10:33,051 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:10:33,052 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:10:33,428 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:33,429 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:10:33,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:34,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:35,429 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:35,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:36,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:37,081 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:37,430 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:37,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:38,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:39,431 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:39,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:40,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:41,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:42,467 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:10:42,468 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:42,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:42,792 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:42,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:43,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:44,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:45,863 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:46,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:47,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:47,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:48,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:49,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:50,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:51,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:52,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:52,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:53,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:54,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:55,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:56,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:57,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:57,791 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:57,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:57,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:58,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:59,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:00,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:01,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:02,867 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:03,799 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:03,867 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:04,871 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:05,329 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:11:05,329 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 with policy now +2025-01-06 17:11:05,332 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:11:05,332 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:05,332 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:05,333 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:11:05,580 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/eqwxx7lq-media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,843 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/niwyj3l8-media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,877 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:06,881 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:07,446 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:07,875 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:08,875 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:09,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:09,878 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:10,879 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:11,447 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:11,872 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:12,468 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:11:12,810 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:12,810 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:12,879 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:13,448 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:13,878 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:14,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:14,928 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:15,885 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:16,886 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:17,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:18,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:19,887 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:20,803 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:20,886 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:21,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:22,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:23,888 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:24,451 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:11:24,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:25,115 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:25,115 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:25,116 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:25,452 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:25,452 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:25,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:26,281 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:26,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:27,453 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:27,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:27,794 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:27,894 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:28,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:29,453 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:29,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:30,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:31,454 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:31,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:32,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:33,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:34,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:35,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:36,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:36,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:37,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:38,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:39,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:40,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:41,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:42,468 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:11:42,468 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:42,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:42,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:42,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:43,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:44,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:45,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:46,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:47,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:47,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:48,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:49,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:50,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:51,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:52,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:52,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:53,462 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:53,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:54,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:55,900 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:56,402 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:11:56,402 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 with policy now +2025-01-06 17:11:56,404 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:11:56,405 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:56,405 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:56,405 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:11:56,669 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/3a2q13s4-media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:56,896 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:56,931 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/y3uhn7yv-media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:57,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:57,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:57,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:57,884 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:57,897 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:58,900 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:59,909 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:00,914 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:01,465 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:01,911 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:02,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:03,465 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:03,812 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:03,920 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:04,919 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:05,466 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:05,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:06,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:07,925 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:08,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:08,924 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:09,923 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:10,935 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:11,933 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:12,469 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:12:12,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:12:12,802 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:12:12,927 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:13,170 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:13,170 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:13,172 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:13,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:13,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:13,560 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:12:13,560 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:12:13,724 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/t0smfsyv-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:13,761 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:12:13,921 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:14,115 INFO wandb-upload_0:2085739 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu2qocgvx +2025-01-06 17:12:14,150 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:12:14,159 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval +2025-01-06 17:12:14,528 INFO SenderThread:2085739 [sender.py:send_request_log_artifact():1525] logged artifact run-8ycpgp3n-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjI0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwODU0NA==', 'latestArtifact': None}} +2025-01-06 17:12:14,528 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:12:14,528 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 17:12:14,528 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:14,529 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:14,529 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,529 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,530 DEBUG SenderThread:2085739 [sender.py:send():379] send: telemetry +2025-01-06 17:12:14,530 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,533 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,534 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,534 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,535 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,535 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,536 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,537 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,537 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,538 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,538 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,539 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,539 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,540 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,541 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,541 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,542 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,542 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,543 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,543 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,544 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,544 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,545 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:12:14,724 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/fscaesmc-media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:14,743 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/en3kdbz6-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:14,917 INFO wandb-upload_0:2085739 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw5ug3ik4 +2025-01-06 17:12:14,921 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:14,981 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/w01m86g1-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:15,282 INFO SenderThread:2085739 [sender.py:send_request_log_artifact():1525] logged artifact run-8ycpgp3n-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjI0Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwODU0Nw==', 'latestArtifact': None}} +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:12:15,282 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json with policy now +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:15,283 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:15,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:15,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:15,513 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/7kw87lln-media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:15,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:16,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:17,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:18,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:19,812 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:19,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:20,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:21,706 DEBUG SenderThread:2085739 [sender.py:send():379] send: exit +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():588] handling runtime: 279 +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():594] send defer +2025-01-06 17:12:21,706 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,706 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:12:21,707 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:12:21,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:12:21,707 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:12:21,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:12:21,707 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:12:21,707 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:12:21,708 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:12:21,709 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:12:21,760 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:12:21,760 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:12:21,761 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:12:21,761 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,761 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:12:21,761 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:12:21,761 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:12:21,761 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,762 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:12:21,762 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,762 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:12:21,762 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:12:21,762 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,762 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:12:22,473 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:23,474 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:28,178 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:28,178 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:28,179 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,180 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:12:28,180 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:28,180 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,180 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:28,181 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:28,181 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:28,181 DEBUG SenderThread:2085739 [sender.py:send():379] send: summary +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:28,185 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:12:28,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,185 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:12:28,186 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,186 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:12:28,190 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:28,299 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:12:28,299 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,299 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:12:28,299 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,299 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:12:28,476 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:12:28,476 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:28,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:30,823 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:12:30,823 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:30,823 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:30,824 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:12:30,824 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:30,824 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:12:30,824 INFO SenderThread:2085739 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:12:30,826 INFO SenderThread:2085739 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:12:30,828 INFO SenderThread:2085739 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:12:30,838 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:12:30,839 DEBUG SenderThread:2085739 [sender.py:send():379] send: artifact +2025-01-06 17:12:30,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:30,842 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:12:31,327 INFO wandb-upload_0:2085739 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn6ah3rw0 +2025-01-06 17:12:31,327 INFO wandb-upload_1:2085739 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpros8k_3v +2025-01-06 17:12:31,477 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:31,491 INFO wandb-upload_2:2085739 [upload_job.py:push():88] Uploaded file /tmp/tmp3o0ybe06/wandb-job.json +2025-01-06 17:12:31,708 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'versionIndex': 421}}} +2025-01-06 17:12:32,044 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml config.yaml +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json wandb-summary.json +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt requirements.txt +2025-01-06 17:12:32,479 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:12:32,479 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:32,481 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log output.log +2025-01-06 17:12:32,483 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch diff.patch +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:12:32,485 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:32,485 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,485 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:12:32,488 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,488 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:12:32,488 INFO SenderThread:2085739 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:12:32,657 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:12:32,675 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt +2025-01-06 17:12:32,679 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:32,708 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:32,709 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:32,710 INFO wandb-upload_4:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:32,787 INFO wandb-upload_3:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/yx6fcvjx-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:32,988 INFO Thread-11 (_thread_body):2085739 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:12:32,988 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,988 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:12:32,988 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,988 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:12:32,988 INFO SenderThread:2085739 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:12:32,990 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,990 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:12:32,990 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:12:33,184 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:33,184 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:12:33,184 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send():379] send: final +2025-01-06 17:12:33,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:33,185 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send():379] send: footer +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:33,185 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:12:33,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:33,186 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:33,186 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:12:33,186 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:33,187 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:12:33,187 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:12:33,187 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:12:33,189 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:33,257 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:12:33,259 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:12:33,259 INFO HandlerThread:2085739 [handler.py:finish():882] shutting down handler +2025-01-06 17:12:34,186 INFO WriterThread:2085739 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [sender.py:finish():1615] shutting down sender +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [file_pusher.py:join():175] waiting for file pusher diff --git a/8ycpgp3n/wandb/debug.log b/8ycpgp3n/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4b0322378317a98332399ea8c83996a700bd7b6f --- /dev/null +++ b/8ycpgp3n/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Configure stats pid to 2085574 +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/settings +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug.log +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug-internal.log +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n'} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():619] starting backend +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():623] setting up manager +2025-01-06 17:07:42,086 INFO MainThread:2085574 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:07:42,087 INFO MainThread:2085574 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:07:42,098 INFO MainThread:2085574 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:07:42,106 INFO MainThread:2085574 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:07:42,383 INFO MainThread:2085574 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:07:42,451 INFO MainThread:2085574 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:07:42,452 INFO MainThread:2085574 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:07:42,791 INFO MainThread:2085574 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:07:42,792 INFO MainThread:2085574 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/tb, True +2025-01-06 17:07:47,331 INFO MainThread:2085574 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:12:34,259 WARNING MsgRouterThr:2085574 [router.py:message_loop():77] message_loop has been closed diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08fa2055484f4ed557c658630adcdd4018b61e42 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736201262 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e671a6ef54925430a67da569dc7048d49af892b6 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[6, 10, 0.6, 0.23432976012774953, 0.8932412265944779, 0.30846063796536327, 0.8765580951414395, 0.3514861315785843, 0.812301307854181, 0.36777679624817783, 0.8060191310330135]]} \ No newline at end of file diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e2c3d9d3000541613cf04aad14e0bf73d098263a --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b356ff1064e631440b303c289ad5129b415f5b66f6e26b121c73e79ab327791 +size 1108466 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab01dd6abf3848cef5f383825ef41461bede7965 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2cbfc725296a7e599308bb92b98fc4f7503e9925f167359530a11b999bc2ac +size 1190976 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ca9953f822c7c421df50b06d4a62d58099e1e3ea --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff803fd511eb33dc17ee397bacd4f0752824130ab07d7f6040f228079992066e +size 1202106 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..36349892bddf4e74c620193d4812ccec4b9ed861 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:509210f2f4eb425aa9519dd8da692096f5f7b5e8b5cbfa9aa09fef0f4a71f5e8 +size 1236550 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c038c4f7dd75826168074979044132fc2b36cd6 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:195f521b8e7cf2a9579a5250a3c557c81dd59fd9cc49166add462c0498d56180 +size 1143381 diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ac9915417ae8f42a3dfdc56a6beb418eb86d14a0 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log @@ -0,0 +1,272 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736201267.332328]: resetting environment +[INFO] [1736201267.337207]: cleared current plan +[INFO] [1736201267.338579]: moving to home +[INFO] [1736201269.341276]: reached home +[INFO] [1736201269.342680]: reset action history +[INFO] [1736201272.050559]: environment reset +Starting episode 0 +[INFO] [1736201272.052741]: resetting environment +[INFO] [1736201272.055928]: cleared current plan +[INFO] [1736201272.056683]: moving to home +[INFO] [1736201273.059056]: reached home +[INFO] [1736201273.060885]: reset action history +[INFO] [1736201275.769307]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736201294.323352]: resetting environment +[INFO] [1736201294.326187]: cleared current plan +[INFO] [1736201294.327032]: moving to home +[INFO] [1736201295.329941]: reached home +[INFO] [1736201295.331458]: reset action history +[INFO] [1736201298.038713]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/episode_rosbags/episode_0_2025-01-06-17-08-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2085574.log +started roslaunch server http://158.130.50.37:42533/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2085840] +process[print_text-2]: started with pid [2085864] +process[rosbag_record-3]: started with pid [2085865] +started bagging! +For topic gripper_width: timestamp difference is -218412503 for nearest: 1736201322940155402 - target: 1736201323158567905 at idx 249 +[INFO] [1736201325.043944]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736201325.784019]: Stored episode 1. +[INFO] [1736201325.785554]: Saving video... +[INFO] [1736201326.003607]: Video saved! +Starting episode 1 +[INFO] [1736201326.015397]: resetting environment +[INFO] [1736201326.025756]: cleared current plan +[INFO] [1736201326.025999]: moving to home +[INFO] [1736201329.628540]: reached home +[INFO] [1736201329.629060]: reset action history +[INFO] [1736201332.336983]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201334.630624]: resetting environment +[INFO] [1736201334.632778]: cleared current plan +[INFO] [1736201334.632934]: moving to home +[INFO] [1736201335.634460]: reached home +[INFO] [1736201335.634881]: reset action history +[INFO] [1736201338.341823]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/episode_rosbags/episode_1_2025-01-06-17-08-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2085574.log +started roslaunch server http://158.130.50.37:41119/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2085975] +process[print_text-5]: started with pid [2085976] +process[rosbag_record-6]: started with pid [2085977] +started bagging! +For topic gripper_width: timestamp difference is -101404829 for nearest: 1736201343940165834 - target: 1736201344041570663 at idx 85 +For topic gripper_width: timestamp difference is 105562247 for nearest: 1736201352940142430 - target: 1736201352834580183 at idx 205 +For topic gripper_width: timestamp difference is 106185000 for nearest: 1736201353740140002 - target: 1736201353633955002 at idx 212 +[INFO] [1736201365.296550]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736201366.061653]: Stored episode 2. +[INFO] [1736201366.061900]: Saving video... +[WARN] [1736201366.645574]: Plan exhausted +[INFO] [1736201366.581128]: Video saved! +Starting episode 2 +[INFO] [1736201366.636749]: resetting environment +[INFO] [1736201366.647446]: cleared current plan +[INFO] [1736201366.649206]: moving to home +[INFO] [1736201370.053521]: reached home +[INFO] [1736201370.057250]: reset action history +[INFO] [1736201372.843372]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201382.257498]: resetting environment +[INFO] [1736201382.264601]: cleared current plan +[INFO] [1736201382.264860]: moving to home +[INFO] [1736201383.266549]: reached home +[INFO] [1736201383.267065]: reset action history +[INFO] [1736201385.974740]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/episode_rosbags/episode_2_2025-01-06-17-09-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2085574.log +started roslaunch server http://158.130.50.37:40861/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2086134] +process[print_text-8]: started with pid [2086135] +process[rosbag_record-9]: started with pid [2086136] +started bagging! +[INFO] [1736201413.129655]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736201413.904214]: Stored episode 3. +[INFO] [1736201413.907619]: Saving video... +[WARN] [1736201414.485754]: Plan exhausted +[WARN] [1736201414.535954]: Plan exhausted +[WARN] [1736201414.567736]: Plan exhausted +[WARN] [1736201414.605905]: Plan exhausted +[WARN] [1736201414.645923]: Plan exhausted +[INFO] [1736201414.596822]: Video saved! +Starting episode 3 +[INFO] [1736201414.642556]: resetting environment +[INFO] [1736201414.658071]: cleared current plan +[INFO] [1736201414.658758]: moving to home +[INFO] [1736201418.065496]: reached home +[INFO] [1736201418.069093]: reset action history +[INFO] [1736201420.846584]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201432.124236]: resetting environment +[INFO] [1736201432.136052]: cleared current plan +[INFO] [1736201432.142076]: moving to home +[INFO] [1736201433.148546]: reached home +[INFO] [1736201433.152006]: reset action history +[INFO] [1736201436.040270]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/episode_rosbags/episode_3_2025-01-06-17-10-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2085574.log +started roslaunch server http://158.130.50.37:35395/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2086270] +process[print_text-11]: started with pid [2086271] +process[rosbag_record-12]: started with pid [2086272] +started bagging! +[INFO] [1736201464.075230]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736201464.855035]: Stored episode 4. +[INFO] [1736201464.857815]: Saving video... +[INFO] [1736201465.288044]: Video saved! +Starting episode 4 +[INFO] [1736201465.332937]: resetting environment +[INFO] [1736201465.342762]: cleared current plan +[INFO] [1736201465.345582]: moving to home +[INFO] [1736201468.752021]: reached home +[INFO] [1736201468.760097]: reset action history +[INFO] [1736201471.536216]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736201484.211960]: resetting environment +[INFO] [1736201484.222329]: cleared current plan +[INFO] [1736201484.229250]: moving to home +[INFO] [1736201485.234566]: reached home +[INFO] [1736201485.239653]: reset action history +[INFO] [1736201488.013418]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/episode_rosbags/episode_4_2025-01-06-17-11-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2085574.log +started roslaunch server http://158.130.50.37:44305/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2086405] +process[print_text-14]: started with pid [2086406] +process[rosbag_record-15]: started with pid [2086407] +started bagging! +For topic gripper_width: timestamp difference is -265330898 for nearest: 1736201512240164889 - target: 1736201512505495787 at idx 249 +[INFO] [1736201515.172670]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736201515.933274]: Stored episode 5. +[INFO] [1736201515.934773]: Saving video... +[INFO] [1736201516.342563]: Video saved! +[INFO] [1736201516.404959]: resetting environment +[INFO] [1736201516.428871]: cleared current plan +[INFO] [1736201516.432247]: moving to home +[INFO] [1736201519.837102]: reached home +[INFO] [1736201519.852655]: reset action history +[INFO] [1736201522.649285]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d7cf694c22193a63e5137b4f51b6180956feca21 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:07:42.474816", + "startedAt": "2025-01-06T22:07:42.079437", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.0075, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.117, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.969, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 314.2787742614746 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d0c82e84ed36a99ed4d313a18f365d88f7c87878 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6b356ff1064e631440b303c289ad5129b415f5b66f6e26b121c73e79ab327791", "size": 1108466, "path": "media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4"}, "global_step": 665, "_timestamp": 1736201534.0554848, "_runtime": 271.967661857605, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ca2cbfc725296a7e599308bb92b98fc4f7503e9925f167359530a11b999bc2ac", "size": 1190976, "path": "media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4"}, "eval/num_success": 1.0, "episode": 10.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "ff803fd511eb33dc17ee397bacd4f0752824130ab07d7f6040f228079992066e", "size": 1202106, "path": "media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "509210f2f4eb425aa9519dd8da692096f5f7b5e8b5cbfa9aa09fef0f4a71f5e8", "size": 1236550, "path": "media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "195f521b8e7cf2a9579a5250a3c557c81dd59fd9cc49166add462c0498d56180", "size": 1143381, "path": "media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://n62s8291zutuxh313nqm2th9oh48yyf195uvpfq355wjwsy1jyqjwxz10ckb07tmqwco8qzuo1a3ixwrge201x2lsv7af3px7ojwo1pl0eernj4xerxajfm08j7tq4q2/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://spjru4fu7wyuho1pymv53hq9fyhot1oykanle81dvwr1jmaruwsys87nu41lhduqoyqcpieemz2xaa40nc0eamwz4xjml607gjn6wlivuqnmtkj2aapqspt91exd2a0p:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 6, "total_num_episodes": 10, "total_success_rate": 0.6, "total_success_rate_lb_0.01": 0.23432976012774953, "total_success_rate_ub_0.01": 0.8932412265944779, "total_success_rate_lb_0.025": 0.30846063796536327, "total_success_rate_ub_0.025": 0.8765580951414395, "total_success_rate_lb_0.05": 0.3514861315785843, "total_success_rate_ub_0.05": 0.812301307854181, "total_success_rate_lb_0.1": 0.36777679624817783, "total_success_rate_ub_0.1": 0.8060191310330135, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f0bc1a89c9e598fde9e4540bf9021bb7fda1042ae9d2788369c331e7b0bce208", "size": 506, "artifact_path": "wandb-client-artifact://q891v3s1nnjic3kxqyyr2mpsx8bq5d7m59mimgns0gbodla16df78dta6qgo0tzcmwye3q20xoztngtepolekgrdlal67hnzy6wu98z830q6t99henx4spox65kl8ow4/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://l4xyooxrhlf6wh8mk50jhwzfevjl9jqs8dte2lhsl21nimdlc61j71s3b0w06x8doopw5kaj3s3rletf4dxldsywl09j44mt8m20yks64zcc5rn1xuwqab9k6274bm4z:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 279}, "eval/total_success_rate": 0.6000000238418579, "eval/total_success_rate_lb_0.01": 0.23432976007461548, "eval/total_success_rate_ub_0.01": 0.8932412266731262, "eval/total_success_rate_lb_0.025": 0.30846065282821655, "eval/total_success_rate_ub_0.025": 0.8765580654144287, "eval/total_success_rate_lb_0.05": 0.3514861464500427, "eval/total_success_rate_ub_0.05": 0.8123012781143188, "eval/total_success_rate_lb_0.1": 0.3677768111228943, "eval/total_success_rate_ub_0.1": 0.806019127368927} \ No newline at end of file diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug-internal.log b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f9535781ac7a77a78ebd574163881fc0f9275b2 --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-06 17:07:42,087 INFO StreamThr :2085739 [internal.py:wandb_internal():85] W&B internal server running at pid: 2085739, started at: 2025-01-06 17:07:42.087026 +2025-01-06 17:07:42,088 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:07:42,099 INFO WriterThread:2085739 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb +2025-01-06 17:07:42,099 DEBUG SenderThread:2085739 [sender.py:send():379] send: header +2025-01-06 17:07:42,108 DEBUG SenderThread:2085739 [sender.py:send():379] send: run +2025-01-06 17:07:42,367 INFO SenderThread:2085739 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files +2025-01-06 17:07:42,367 INFO SenderThread:2085739 [sender.py:_start_run_threads():1188] run started: 8ycpgp3n with start time 1736201262.087823 +2025-01-06 17:07:42,383 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:07:42,384 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:07:42,456 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:07:42,461 DEBUG HandlerThread:2085739 [system_info.py:__init__():26] System info init +2025-01-06 17:07:42,461 DEBUG HandlerThread:2085739 [system_info.py:__init__():41] System info init done +2025-01-06 17:07:42,461 INFO HandlerThread:2085739 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:07:42,461 INFO SystemMonitor:2085739 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:07:42,461 INFO HandlerThread:2085739 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:07:42,462 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:07:42,463 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:07:42,463 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:07:42,464 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:07:42,464 INFO SystemMonitor:2085739 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:07:42,474 DEBUG HandlerThread:2085739 [system_info.py:probe():152] Probing system +2025-01-06 17:07:42,476 DEBUG HandlerThread:2085739 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:probe():200] Probing system done +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:07:42.474816', 'startedAt': '2025-01-06T22:07:42.079437', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.0075, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.117, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.969, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 314.2787742614746}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:07:42,481 INFO HandlerThread:2085739 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:07:42,481 INFO HandlerThread:2085739 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:07:42,481 DEBUG HandlerThread:2085739 [system_info.py:_save_code():44] Saving code +2025-01-06 17:07:42,482 DEBUG HandlerThread:2085739 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:07:42,482 DEBUG HandlerThread:2085739 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:07:42,544 DEBUG HandlerThread:2085739 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:07:42,544 INFO HandlerThread:2085739 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:07:42,547 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:07:42,547 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:07:42,548 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:07:42,548 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:07:42,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:07:42,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:07:42,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:07:42,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:07:42,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:42,792 INFO HandlerThread:2085739 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/tb" + save: true +} +control { + relay_id: "f007897853a543cc8f6f92970a65dbe3" +} +_info { + stream_id: "8ycpgp3n" +} + +2025-01-06 17:07:42,792 DEBUG HandlerThread:2085739 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:07:42,799 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/hgz8a79o-wandb-metadata.json +2025-01-06 17:07:42,799 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/m8dd2rbx-diff.patch +2025-01-06 17:07:42,848 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/mplv54em-code/FISH/eval_robot.py +2025-01-06 17:07:42,877 DEBUG SenderThread:2085739 [sender.py:send():379] send: telemetry +2025-01-06 17:07:42,877 DEBUG SenderThread:2085739 [sender.py:send():379] send: tbrecord +2025-01-06 17:07:42,878 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:07:42,878 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 with policy live +2025-01-06 17:07:43,070 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/bikec69p-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:07:43,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH +2025-01-06 17:07:43,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code +2025-01-06 17:07:43,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:44,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:45,368 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:45,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:46,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:47,332 DEBUG SenderThread:2085739 [sender.py:send():379] send: config +2025-01-06 17:07:47,332 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:47,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:48,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:49,369 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:49,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:50,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:51,370 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:51,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:52,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:52,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:53,371 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:53,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:54,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:55,371 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:55,813 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:56,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:57,372 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:07:57,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:07:57,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:07:57,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:57,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:07:58,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:07:59,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:00,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:01,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:02,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:03,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:03,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:04,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:05,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:06,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:07,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:08,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:08,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:09,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:10,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:11,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:12,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:12,789 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:12,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:13,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:13,939 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:14,376 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:08:14,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:15,377 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:15,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:16,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:17,377 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:17,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:18,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:19,326 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:19,381 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:19,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:20,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:21,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:22,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:23,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:24,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:24,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:25,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:26,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:27,789 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:27,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:27,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:28,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:29,807 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:29,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:30,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:31,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:32,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:33,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:34,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:35,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:35,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:36,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:37,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:38,808 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:39,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:40,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:40,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:41,809 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:42,465 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:08:42,465 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:08:42,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:42,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:42,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:43,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:44,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:45,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:45,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:46,004 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:46,049 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:08:46,050 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 with policy now +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:08:46,050 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:08:46,051 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:08:46,310 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/oiocex7m-media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:08:46,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos +2025-01-06 17:08:46,665 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/igx7hdy1-media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:08:46,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:47,388 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:47,816 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:48,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:49,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:50,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:51,390 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:51,669 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:51,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:52,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:53,390 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:53,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:54,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:55,391 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:08:55,646 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/s92ufxho-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:08:55,817 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:56,677 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:08:56,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:57,392 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:57,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:08:57,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:08:57,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:58,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:08:59,393 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:08:59,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:00,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:01,394 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:01,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:02,589 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:02,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:03,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:04,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:05,395 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:05,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:06,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:07,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:07,819 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:08,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:09,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:10,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:11,820 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:12,466 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:09:12,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:12,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:12,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:12,935 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:13,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:14,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:15,398 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:15,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:16,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:17,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:18,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:18,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:19,821 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:20,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:21,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:22,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:23,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:24,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:24,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:25,823 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:26,627 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:09:26,627 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 with policy now +2025-01-06 17:09:26,632 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:09:26,633 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:09:26,633 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:09:26,634 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:09:26,822 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:26,894 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/x1fr96t6-media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,402 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:09:27,688 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/yjfmt6dg-media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:09:27,799 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:27,799 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:27,841 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:28,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:29,403 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:29,830 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:29,930 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:30,831 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:31,404 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:31,833 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:32,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:33,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:34,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:35,405 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:35,834 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:35,885 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:36,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:37,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:38,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:39,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:40,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:41,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:41,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:42,408 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:09:42,466 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:09:42,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:42,791 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:42,835 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:43,217 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:09:43,218 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:09:43,218 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:09:43,408 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:43,409 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:09:43,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:44,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:45,409 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:45,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:46,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:47,410 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:09:47,428 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:47,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:48,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:49,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:50,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:51,836 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:52,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:52,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:53,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:54,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:55,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:56,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:57,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:09:57,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:09:57,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:58,220 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:09:58,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:09:59,837 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:00,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:01,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:02,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:03,795 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:03,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:04,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:05,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:06,838 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:07,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:08,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:09,796 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:09,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:10,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:11,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:12,467 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:10:12,790 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:12,790 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:12,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:13,842 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:14,633 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:10:14,633 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 with policy now +2025-01-06 17:10:14,636 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:10:14,637 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:10:14,637 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:10:14,637 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:10:14,840 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:14,928 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/6tlyz576-media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,419 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:10:15,659 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:15,714 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/e2j46bsj-media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:10:15,841 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:16,842 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:17,420 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:17,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:18,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:19,421 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:19,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:20,847 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:20,849 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:21,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:22,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:23,422 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:23,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:24,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:25,851 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:26,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:26,852 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:27,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:27,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:27,854 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:28,853 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:29,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:30,859 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:31,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:31,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:32,428 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:10:32,678 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/g35elcwn-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:10:32,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:33,051 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:10:33,051 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:10:33,052 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:10:33,428 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:33,429 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:10:33,855 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:34,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:35,429 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:35,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:36,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:37,081 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:37,430 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:37,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:38,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:39,431 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:10:39,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:40,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:41,856 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:42,467 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:10:42,468 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:42,792 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:42,792 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:42,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:43,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:44,857 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:45,863 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:46,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:47,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:47,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:48,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:49,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:50,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:51,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:52,798 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:52,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:53,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:54,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:55,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:56,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:57,791 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:10:57,791 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:10:57,864 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:10:57,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:58,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:10:59,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:00,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:01,865 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:02,867 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:03,799 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:03,867 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:04,871 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:05,329 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:11:05,329 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 with policy now +2025-01-06 17:11:05,332 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:11:05,332 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:05,332 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:05,333 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,445 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:11:05,580 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/eqwxx7lq-media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,843 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/niwyj3l8-media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:11:05,877 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:06,881 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:07,446 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:07,875 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:08,875 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:09,797 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:09,878 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:10,879 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:11,447 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:11,872 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:12,468 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:11:12,810 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:12,810 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:12,879 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:13,448 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:13,878 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:14,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:14,928 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:15,885 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:16,886 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:17,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:18,882 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:19,887 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:20,803 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:20,886 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:21,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:22,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:23,888 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:24,451 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:11:24,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:25,115 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:25,115 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:25,116 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:25,452 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:25,452 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:25,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:26,281 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:26,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:27,453 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:27,794 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:27,794 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:27,894 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:28,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:29,453 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:29,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:30,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:31,454 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:31,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:32,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:33,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:34,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:35,889 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:36,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:36,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:37,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:38,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:39,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:40,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:41,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:42,468 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:11:42,468 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:42,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:42,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:42,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:43,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:44,890 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:45,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:46,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:47,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:47,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:48,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:49,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:50,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:51,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:52,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:52,891 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:53,462 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:53,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:54,892 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:55,900 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:56,402 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:11:56,402 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 with policy now +2025-01-06 17:11:56,404 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:11:56,405 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:11:56,405 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:11:56,405 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:56,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval +2025-01-06 17:11:56,669 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/3a2q13s4-media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:56,896 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:56,931 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/y3uhn7yv-media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:11:57,463 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:11:57,793 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:11:57,793 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:11:57,884 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:11:57,897 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:58,900 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:11:59,909 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:00,914 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:01,465 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:01,911 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:02,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:03,465 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:03,812 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:03,920 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:04,919 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:05,466 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:05,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:06,917 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:07,925 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:08,818 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:08,924 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:09,923 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:10,935 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:11,933 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:12,469 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:12:12,802 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:12:12,802 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:12:12,927 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:13,170 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:13,170 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:13,172 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:13,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:13,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:13,560 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:12:13,560 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:12:13,724 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/t0smfsyv-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:13,761 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:12:13,921 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:14,115 INFO wandb-upload_0:2085739 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu2qocgvx +2025-01-06 17:12:14,150 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:12:14,159 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:14,470 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table +2025-01-06 17:12:14,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval +2025-01-06 17:12:14,528 INFO SenderThread:2085739 [sender.py:send_request_log_artifact():1525] logged artifact run-8ycpgp3n-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjI0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwODU0NA==', 'latestArtifact': None}} +2025-01-06 17:12:14,528 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:12:14,528 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 17:12:14,528 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:14,529 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:14,529 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,529 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,530 DEBUG SenderThread:2085739 [sender.py:send():379] send: telemetry +2025-01-06 17:12:14,530 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,533 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,534 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,534 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,535 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,535 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,536 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,537 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,537 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,538 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,538 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,539 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,539 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,540 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,541 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,541 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,542 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,542 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,543 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,543 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,544 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:14,544 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:14,545 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:12:14,724 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/fscaesmc-media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:14,743 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/en3kdbz6-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:14,917 INFO wandb-upload_0:2085739 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw5ug3ik4 +2025-01-06 17:12:14,921 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:14,981 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/w01m86g1-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:15,282 INFO SenderThread:2085739 [sender.py:send_request_log_artifact():1525] logged artifact run-8ycpgp3n-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjI0Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUwODU0Nw==', 'latestArtifact': None}} +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send():379] send: files +2025-01-06 17:12:15,282 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json with policy now +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:15,282 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:15,283 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:15,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:15,471 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:15,513 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/7kw87lln-media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:15,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:16,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:17,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:18,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:19,812 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:19,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:20,922 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:21,706 DEBUG SenderThread:2085739 [sender.py:send():379] send: exit +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():588] handling runtime: 279 +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:21,706 INFO SenderThread:2085739 [sender.py:send_exit():594] send defer +2025-01-06 17:12:21,706 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,706 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:12:21,707 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:12:21,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:12:21,707 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:12:21,707 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:12:21,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:12:21,707 INFO HandlerThread:2085739 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:12:21,707 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:12:21,707 DEBUG SystemMonitor:2085739 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:12:21,708 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:12:21,709 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:12:21,760 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:12:21,760 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:12:21,761 INFO HandlerThread:2085739 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:12:21,761 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,761 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:12:21,761 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:12:21,761 DEBUG SenderThread:2085739 [sender.py:send():379] send: stats +2025-01-06 17:12:21,761 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,762 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:12:21,762 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:21,762 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:12:21,762 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:12:21,762 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:21,762 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:12:22,473 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:23,474 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:28,178 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:28,178 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:28,179 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,180 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:12:28,180 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:28,180 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,180 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:12:28,180 DEBUG SenderThread:2085739 [sender.py:send():379] send: history +2025-01-06 17:12:28,181 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:12:28,181 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:28,181 DEBUG SenderThread:2085739 [sender.py:send():379] send: summary +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:12:28,185 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:12:28,185 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:12:28,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,185 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:12:28,186 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,186 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:12:28,190 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:12:28,299 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:12:28,299 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:28,299 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:12:28,299 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:28,299 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:12:28,476 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:12:28,476 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:28,707 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:30,823 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:12:30,823 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:30,823 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:30,824 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:12:30,824 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:30,824 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:12:30,824 INFO SenderThread:2085739 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:12:30,826 INFO SenderThread:2085739 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:12:30,828 INFO SenderThread:2085739 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:12:30,838 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:12:30,839 DEBUG SenderThread:2085739 [sender.py:send():379] send: artifact +2025-01-06 17:12:30,839 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:30,842 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:12:31,327 INFO wandb-upload_0:2085739 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn6ah3rw0 +2025-01-06 17:12:31,327 INFO wandb-upload_1:2085739 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpros8k_3v +2025-01-06 17:12:31,477 INFO Thread-12 :2085739 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:31,491 INFO wandb-upload_2:2085739 [upload_job.py:push():88] Uploaded file /tmp/tmp3o0ybe06/wandb-job.json +2025-01-06 17:12:31,708 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTg5Nzc3Mg==', 'versionIndex': 421}}} +2025-01-06 17:12:32,044 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:12:32,044 INFO SenderThread:2085739 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml config.yaml +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json wandb-summary.json +2025-01-06 17:12:32,478 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt requirements.txt +2025-01-06 17:12:32,479 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:12:32,479 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:32,481 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log output.log +2025-01-06 17:12:32,483 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/diff.patch diff.patch +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 media/videos/eval/1_eval_1_ca2cbfc725296a7e5993.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 media/videos/eval/0_eval_0_6b356ff1064e631440b3.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 media/videos/eval/4_eval_4_195f521b8e7cf2a9579a.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 media/videos/eval/2_eval_2_ff803fd511eb33dc17ee.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 media/videos/eval/3_eval_3_509210f2f4eb425aa951.mp4 +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json media/table/eval/total_success_rate_ci_6_f0bc1a89c9e598fde9e4.table.json +2025-01-06 17:12:32,484 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:12:32,485 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:32,485 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,485 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:12:32,488 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,488 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:12:32,488 INFO SenderThread:2085739 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:12:32,657 INFO wandb-upload_0:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/config.yaml +2025-01-06 17:12:32,675 INFO wandb-upload_2:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/requirements.txt +2025-01-06 17:12:32,679 INFO wandb-upload_1:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/wandb-summary.json +2025-01-06 17:12:32,708 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:32,709 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:32,710 INFO wandb-upload_4:2085739 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/files/output.log +2025-01-06 17:12:32,787 INFO wandb-upload_3:2085739 [upload_job.py:push():130] Uploaded file /tmp/tmpae5la3hbwandb/yx6fcvjx-events.out.tfevents.1736201262.leonmkim-ROG-Strix-G15CS-G15CS.2085574.0 +2025-01-06 17:12:32,988 INFO Thread-11 (_thread_body):2085739 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:12:32,988 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,988 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:12:32,988 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,988 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:12:32,988 INFO SenderThread:2085739 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:12:32,990 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:32,990 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:12:32,990 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:12:32,990 INFO SenderThread:2085739 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:12:33,184 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:33,184 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:12:33,184 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:12:33,184 INFO SenderThread:2085739 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send():379] send: final +2025-01-06 17:12:33,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:12:33,185 INFO HandlerThread:2085739 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send():379] send: footer +2025-01-06 17:12:33,185 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: defer +2025-01-06 17:12:33,185 INFO SenderThread:2085739 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:12:33,185 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:33,186 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:33,186 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:12:33,186 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:12:33,187 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:12:33,187 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:12:33,187 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:12:33,189 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:12:33,257 DEBUG SenderThread:2085739 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:12:33,258 INFO MainThread:2085739 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:12:33,259 DEBUG HandlerThread:2085739 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:12:33,259 INFO HandlerThread:2085739 [handler.py:finish():882] shutting down handler +2025-01-06 17:12:34,186 INFO WriterThread:2085739 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [sender.py:finish():1615] shutting down sender +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:12:34,258 INFO SenderThread:2085739 [file_pusher.py:join():175] waiting for file pusher diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug.log b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4b0322378317a98332399ea8c83996a700bd7b6f --- /dev/null +++ b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Configure stats pid to 2085574 +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:07:42,081 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/settings +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug.log +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/logs/debug-internal.log +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n'} +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():619] starting backend +2025-01-06 17:07:42,082 INFO MainThread:2085574 [wandb_init.py:init():623] setting up manager +2025-01-06 17:07:42,086 INFO MainThread:2085574 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:07:42,087 INFO MainThread:2085574 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:07:42,098 INFO MainThread:2085574 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:07:42,106 INFO MainThread:2085574 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:07:42,383 INFO MainThread:2085574 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:07:42,451 INFO MainThread:2085574 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:07:42,452 INFO MainThread:2085574 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:07:42,790 INFO MainThread:2085574 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:07:42,791 INFO MainThread:2085574 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:07:42,792 INFO MainThread:2085574 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/8ycpgp3n/tb, True +2025-01-06 17:07:47,331 INFO MainThread:2085574 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:12:34,259 WARNING MsgRouterThr:2085574 [router.py:message_loop():77] message_loop has been closed diff --git a/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb new file mode 100644 index 0000000000000000000000000000000000000000..82d6c1766f4db224a36a9d8d386343bfc121373d Binary files /dev/null and b/8ycpgp3n/wandb/run-20250106_170742-8ycpgp3n/run-8ycpgp3n.wandb differ diff --git a/b3d2fbi3/episode_rosbags/episode_1_2025-01-07-21-45-44.bag b/b3d2fbi3/episode_rosbags/episode_1_2025-01-07-21-45-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..5afebb40aaa19b45b15c08a0d3b724f48c180b4f --- /dev/null +++ b/b3d2fbi3/episode_rosbags/episode_1_2025-01-07-21-45-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:820bd5fb24c822b1cefe5d32be3b0558d93554bdc28a940511f918b101cf5dda +size 1411911863 diff --git a/b3d2fbi3/episode_rosbags/episode_2_2025-01-07-21-46-30.bag b/b3d2fbi3/episode_rosbags/episode_2_2025-01-07-21-46-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..6763d5b9e5b782df12b5b017b39f40c89f7094ae --- /dev/null +++ b/b3d2fbi3/episode_rosbags/episode_2_2025-01-07-21-46-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:979d556d808bd6a8781cd8399b7c697ba9e32cdafb2d2574e364b89b410c9574 +size 1414322755 diff --git a/fnq3cpdm/episode_rosbags/cam_tf_world.npy b/fnq3cpdm/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/fnq3cpdm/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/fnq3cpdm/episode_rosbags/depth_K.npy b/fnq3cpdm/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/fnq3cpdm/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/fnq3cpdm/eval_robot.log b/fnq3cpdm/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..462d6eac35baf77ec12464056ed8d708b0f438dd --- /dev/null +++ b/fnq3cpdm/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:12:45,307][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:12:45,310][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kccehv5k/.hydra/config.yaml b/kccehv5k/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/kccehv5k/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kccehv5k/.hydra/hydra.yaml b/kccehv5k/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c09773acc19873dc69205ce1110d895fc5ab3d5 --- /dev/null +++ b/kccehv5k/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kccehv5k/.hydra/overrides.yaml b/kccehv5k/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kccehv5k/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kccehv5k/episode_rosbags/aligned_depth_to_color_K.npy b/kccehv5k/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kccehv5k/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kccehv5k/episode_rosbags/cam_tf_world.npy b/kccehv5k/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kccehv5k/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kccehv5k/episode_rosbags/color_K.npy b/kccehv5k/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kccehv5k/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kccehv5k/episode_rosbags/depth_K.npy b/kccehv5k/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kccehv5k/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kccehv5k/eval_robot.log b/kccehv5k/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..63bdecd8017ca94d6321684b32f8d65ba4dcb2ec --- /dev/null +++ b/kccehv5k/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-31 20:46:16,947][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:586: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-31 20:46:16,950][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:522: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kccehv5k/eval_video/0_eval.mp4 b/kccehv5k/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64a854c20425dfef66eba6f37df29723879a9569 --- /dev/null +++ b/kccehv5k/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c4871aa0a38b24cd6b714c4d872f1bdf15ee9e1368eca5c4c5cca7795db545 +size 1147449 diff --git a/kccehv5k/eval_video/1_eval.mp4 b/kccehv5k/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..478a8702384d36c3b932341e4bf3b9c980617bf0 --- /dev/null +++ b/kccehv5k/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99c821ca0f3dcf058d7bdc57f31c99fec024325049b678d22e60cdfcb7032ce +size 1154686 diff --git a/kccehv5k/eval_video/2_eval.mp4 b/kccehv5k/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1833c678ad3af1637f1656faaccc72b42325eea6 --- /dev/null +++ b/kccehv5k/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497369a7fef0c32f3ee5bafccfc53b7a3889416759469d55aac8abeb7d3c5011 +size 1153780 diff --git a/kccehv5k/eval_video/3_eval.mp4 b/kccehv5k/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68a29cad76ffa17af095886eb7b17779cb1c8e8d --- /dev/null +++ b/kccehv5k/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8beb81bb83090b35f94094b007cb72a5ac5e21914596e2821215154f3398a6 +size 1163442 diff --git a/kccehv5k/eval_video/4_eval.mp4 b/kccehv5k/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea389243cba17f405864fd49527bfbc0e070f74f --- /dev/null +++ b/kccehv5k/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8469a9945cf9312a1f45cb2c0e3bef7b03b0c9263ac88eff33e692616036151 +size 1242957 diff --git a/kccehv5k/tb/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 b/kccehv5k/tb/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 new file mode 100644 index 0000000000000000000000000000000000000000..bcd1db9b58e3ab5ea0b10ee1f58027a751b47ac7 --- /dev/null +++ b/kccehv5k/tb/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53ac2962ccb1b424aa336ad303d630f49d153a37bba4b69c591e85a129d0844f +size 5503 diff --git a/kccehv5k/wandb/debug-internal.log b/kccehv5k/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2780c8030150722974220e42c7507589b190f143 --- /dev/null +++ b/kccehv5k/wandb/debug-internal.log @@ -0,0 +1,1547 @@ +2024-12-31 20:46:23,179 INFO StreamThr :1737239 [internal.py:wandb_internal():85] W&B internal server running at pid: 1737239, started at: 2024-12-31 20:46:23.178761 +2024-12-31 20:46:23,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status +2024-12-31 20:46:23,197 INFO WriterThread:1737239 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb +2024-12-31 20:46:23,197 DEBUG SenderThread:1737239 [sender.py:send():379] send: header +2024-12-31 20:46:23,205 DEBUG SenderThread:1737239 [sender.py:send():379] send: run +2024-12-31 20:46:23,451 INFO SenderThread:1737239 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files +2024-12-31 20:46:23,452 INFO SenderThread:1737239 [sender.py:_start_run_threads():1188] run started: kccehv5k with start time 1735695983.180525 +2024-12-31 20:46:23,475 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 20:46:23,475 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: check_version +2024-12-31 20:46:23,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 20:46:23,553 DEBUG HandlerThread:1737239 [system_info.py:__init__():26] System info init +2024-12-31 20:46:23,554 DEBUG HandlerThread:1737239 [system_info.py:__init__():41] System info init done +2024-12-31 20:46:23,554 INFO HandlerThread:1737239 [system_monitor.py:start():194] Starting system monitor +2024-12-31 20:46:23,554 INFO SystemMonitor:1737239 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 20:46:23,554 INFO HandlerThread:1737239 [system_monitor.py:probe():214] Collecting system info +2024-12-31 20:46:23,555 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 20:46:23,555 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started disk monitoring +2024-12-31 20:46:23,556 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 20:46:23,556 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started memory monitoring +2024-12-31 20:46:23,557 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started network monitoring +2024-12-31 20:46:23,566 DEBUG HandlerThread:1737239 [system_info.py:probe():152] Probing system +2024-12-31 20:46:23,567 DEBUG HandlerThread:1737239 [system_info.py:_probe_git():137] Probing git +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_info.py:_probe_git():145] Probing git done +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_info.py:probe():200] Probing system done +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T01:46:23.566585', 'startedAt': '2025-01-01T01:46:23.167015', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4499.993, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 488.8013000488281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 20:46:23,572 INFO HandlerThread:1737239 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 20:46:23,572 INFO HandlerThread:1737239 [system_monitor.py:probe():227] Publishing system info +2024-12-31 20:46:23,573 DEBUG HandlerThread:1737239 [system_info.py:_save_code():44] Saving code +2024-12-31 20:46:23,574 DEBUG HandlerThread:1737239 [system_info.py:_save_code():67] Saving code done +2024-12-31 20:46:23,574 DEBUG HandlerThread:1737239 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 20:46:23,637 DEBUG HandlerThread:1737239 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 20:46:23,638 INFO HandlerThread:1737239 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 20:46:23,645 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 20:46:23,914 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 20:46:23,915 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: python_packages +2024-12-31 20:46:23,916 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 20:46:23,916 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: stop_status +2024-12-31 20:46:23,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:23,918 INFO HandlerThread:1737239 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/tb" + save: true +} +control { + relay_id: "5f4965b8c2e34f0181637e6010b99766" +} +_info { + stream_id: "kccehv5k" +} + +2024-12-31 20:46:23,919 DEBUG HandlerThread:1737239 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 20:46:23,920 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/6wde7y2z-wandb-metadata.json +2024-12-31 20:46:23,935 INFO wandb-upload_2:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/thrz3lhd-diff.patch +2024-12-31 20:46:23,960 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/gf5ock7d-code/FISH/eval_robot.py +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH +2024-12-31 20:46:24,670 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/z7vm580p-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:46:24,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:25,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:26,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:27,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:28,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:29,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:30,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:31,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:32,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:33,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:34,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:35,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:36,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:37,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:38,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:39,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:40,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:41,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:42,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:43,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:44,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:45,314 DEBUG SenderThread:1737239 [sender.py:send():379] send: telemetry +2024-12-31 20:46:45,314 DEBUG SenderThread:1737239 [sender.py:send():379] send: tbrecord +2024-12-31 20:46:45,314 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:45,315 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:46:45,315 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 with policy live +2024-12-31 20:46:45,315 DEBUG SenderThread:1737239 [sender.py:send():379] send: config +2024-12-31 20:46:45,457 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:45,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:46,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:47,458 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:47,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:48,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:49,459 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:49,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:50,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:51,197 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:51,459 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:51,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:52,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:53,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:54,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:55,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:56,201 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:56,461 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 20:46:56,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:57,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:58,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:59,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:00,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:01,318 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:01,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:02,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:03,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:04,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:05,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:06,319 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:06,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:07,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:08,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:09,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:10,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:11,320 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:11,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:12,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:13,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:14,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:15,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:16,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:16,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:17,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:18,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:19,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:20,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:21,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:21,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:22,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:23,557 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 20:47:23,558 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:47:23,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:24,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:25,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:26,558 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:26,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:27,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:28,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:29,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:30,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:31,559 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:31,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:32,927 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:33,471 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:33,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:34,927 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:35,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:36,715 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:36,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:37,472 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:37,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:38,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:39,472 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:39,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:40,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:41,473 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:41,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:41,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:42,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:43,473 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:43,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:44,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:45,474 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:45,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:46,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:46,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:47,475 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:47,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:48,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:49,475 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:49,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:50,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:51,476 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:51,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:52,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:52,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:53,477 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:53,558 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:47:53,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:54,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:55,477 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:55,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:56,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:57,141 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:57,478 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:57,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:58,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:59,478 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:59,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:00,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:01,479 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:01,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:02,145 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:02,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:03,321 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:48:03,321 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 with policy now +2024-12-31 20:48:03,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:48:03,322 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:48:03,322 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:48:03,323 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:48:03,576 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/bfcifyvh-media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,840 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/5sr4pn21-media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:04,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:05,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:05,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:06,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:07,481 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:07,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:07,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:08,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:09,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:10,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:11,482 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:48:11,483 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:11,761 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/pulmzem4-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:48:11,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:12,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:13,483 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:13,509 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:13,933 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:14,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:15,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:16,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:17,484 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:17,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:18,698 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:18,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:19,485 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:19,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:20,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:21,486 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:21,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:22,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:23,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:48:23,894 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:23,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:24,487 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:24,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:25,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:26,488 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:26,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:27,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:28,488 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:28,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:29,092 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:29,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:30,489 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:30,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:31,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:32,490 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:32,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:33,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:34,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:34,490 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:34,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:35,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:36,491 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:36,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:37,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:38,492 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:38,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:39,487 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:39,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:40,493 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:40,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:41,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:42,493 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:42,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:43,161 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:48:43,161 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:48:43,161 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 with policy now +2024-12-31 20:48:43,162 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:48:43,162 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:48:43,162 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:48:43,424 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/rhfg4oqz-media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:48:43,760 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/jq399vyr-media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,947 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:44,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:44,941 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:45,165 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:45,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:46,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:46,947 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:47,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:48,495 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:48,944 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:49,941 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:50,589 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:50,945 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:51,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:52,496 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:52,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:53,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:48:53,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:54,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:55,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:56,560 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:56,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:57,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:58,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:59,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:00,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:01,560 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:01,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:02,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:03,374 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:03,374 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:03,375 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:03,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:03,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:03,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:04,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:04,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:05,960 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:06,501 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:06,792 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:06,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:07,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:08,502 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:08,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:09,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:10,503 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:10,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:11,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:11,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:12,503 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:12,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:13,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:14,504 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:14,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:15,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:16,505 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:16,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:16,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:17,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:18,505 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:18,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:19,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:20,506 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:20,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:21,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:21,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:22,507 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:22,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:23,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:49:23,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:24,508 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:24,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:25,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:26,509 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:26,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:26,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:27,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:28,510 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:28,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:29,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:30,511 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:30,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:31,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:32,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:32,512 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:32,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:33,962 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:34,512 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:34,798 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:49:34,798 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 with policy now +2024-12-31 20:49:34,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:49:34,802 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:34,802 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:34,802 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:34,963 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:35,093 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/ax5t9rrt-media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:49:35,788 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/136e1m9w-media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,968 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:36,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:36,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:37,824 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:37,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:38,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:39,970 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:40,514 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:40,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:41,968 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:42,515 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:42,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:43,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:43,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:44,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:46,005 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:46,216 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:46,216 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:46,217 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:46,516 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:46,517 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:46,774 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/xkps3yje-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:46,990 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:47,987 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:48,517 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:48,984 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:49,073 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:49,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:50,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:51,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:52,518 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:52,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:53,560 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:49:53,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:54,182 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:54,519 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:54,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:55,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:56,520 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:56,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:57,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:58,521 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:58,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:59,372 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:59,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:00,522 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:00,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:01,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:02,522 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:02,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:03,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:04,384 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:04,523 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:04,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:05,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:06,524 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:06,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:07,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:08,525 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:08,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:09,578 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:09,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:10,525 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:10,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:11,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:12,526 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:12,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:13,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:14,527 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:14,772 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:14,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:15,981 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:16,528 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:16,986 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:17,939 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:50:17,939 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 with policy now +2024-12-31 20:50:17,946 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:50:17,946 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:50:17,946 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:50:17,947 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:50:17,984 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:18,196 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/v104jt0i-media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,528 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:50:18,805 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/6q36fotf-media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,987 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:19,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:19,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:20,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:20,991 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:21,990 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:22,991 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:23,560 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:50:23,995 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:24,531 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:24,998 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:25,505 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:25,999 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:26,531 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:27,004 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:28,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:28,994 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:29,996 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:30,506 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:31,000 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:32,017 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:33,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:34,003 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:35,021 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:35,276 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:50:35,276 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:50:35,277 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:50:35,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:50:35,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:50:35,998 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:36,063 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:36,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:37,019 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:38,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:38,535 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:39,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:40,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:40,536 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:41,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:41,160 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:42,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:42,537 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:43,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:44,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:44,537 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:45,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:46,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:46,198 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:46,538 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:47,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:48,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:48,539 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:49,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:50,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:50,540 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:51,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:51,389 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:52,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:52,540 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:53,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:53,561 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:50:54,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:54,541 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:55,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:56,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:56,542 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:56,589 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:57,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:58,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:58,543 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:59,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:00,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:00,544 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:01,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:01,790 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:02,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:02,544 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:03,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:04,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:04,545 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:05,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:06,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:06,546 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:06,807 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:07,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:08,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:08,546 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:09,023 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:09,345 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:51:09,345 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:51:09,345 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 with policy now +2024-12-31 20:51:09,346 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:51:09,346 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:51:09,346 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:51:09,645 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/ge0ui9g0-media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:09,912 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/g212tycz-media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:10,020 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:10,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:11,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:12,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:12,365 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:13,030 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:14,035 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:14,549 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:15,032 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:16,046 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:17,045 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:17,909 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:18,046 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:18,550 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:19,056 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:20,083 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:21,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:22,054 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:22,910 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:23,056 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:23,561 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:51:24,059 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:25,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:26,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:27,065 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:28,061 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:28,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:29,068 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:30,068 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:31,071 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:32,079 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:33,099 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:33,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:34,083 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:35,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:36,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:37,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:38,091 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:38,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:39,100 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:40,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:41,091 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:42,096 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:43,099 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:43,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:44,112 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:44,361 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:51:44,361 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:51:44,361 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:44,810 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/gknw2j08-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:51:45,108 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:46,115 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:46,560 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:47,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:48,123 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:49,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:49,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:50,129 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:51,122 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:52,137 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:53,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:53,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:51:54,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:54,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:55,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:56,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:57,132 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:58,141 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:59,144 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:59,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:00,149 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:01,144 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:02,158 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:03,157 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:04,159 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:04,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:05,160 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:06,155 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:07,159 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:08,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:09,163 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:09,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:10,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:11,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:12,179 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:13,177 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:14,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:14,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:15,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:16,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:17,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:18,181 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:19,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:19,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:20,184 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:21,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:22,192 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:23,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:23,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:52:24,198 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:25,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:25,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:26,201 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:27,207 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:28,199 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:29,204 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:30,207 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:30,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:31,209 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:32,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:33,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:34,208 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:35,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:35,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:36,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:37,208 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:38,216 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:39,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:40,219 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:40,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:41,221 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:42,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:43,220 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:44,221 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:45,230 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:45,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:46,229 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:47,246 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:48,234 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:49,241 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:50,241 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:50,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:51,242 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:52,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:53,243 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:53,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:52:54,249 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:55,252 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:56,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:56,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:57,245 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:58,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:59,248 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:00,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:01,258 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:01,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:02,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:03,252 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:04,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:05,259 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:06,258 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:06,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:07,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:08,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:09,275 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:10,262 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:11,265 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:11,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:12,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:13,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:14,265 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:15,272 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:16,272 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:16,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:17,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:18,302 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:19,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:20,294 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:21,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:21,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:22,297 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:23,308 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:23,563 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:53:24,307 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:25,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:26,317 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:27,300 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:27,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:28,329 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:29,317 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:30,325 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:31,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:32,330 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:32,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:33,340 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:34,328 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:35,329 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:36,327 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:37,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:37,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:38,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:39,339 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:40,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:41,332 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:42,335 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:42,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:43,336 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:44,336 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:45,346 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:46,346 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:47,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:47,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:48,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:49,345 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:50,349 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:51,354 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:52,348 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:52,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:53,357 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:53,571 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:53:54,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:55,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:56,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:57,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:57,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:58,360 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:59,363 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:00,361 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:01,357 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:02,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:02,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:03,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:04,364 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:05,367 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:06,364 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:07,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:07,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:08,385 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:09,370 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:10,376 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:11,379 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:12,375 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:12,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:13,376 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:14,387 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:15,392 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:16,400 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:17,388 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:17,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:18,393 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:19,386 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:20,398 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:21,392 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:22,399 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:22,574 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:23,397 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:23,564 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:54:24,397 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:25,400 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:26,404 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:27,420 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:28,399 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:28,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:29,405 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:30,402 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:31,409 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:32,413 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:33,421 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:33,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:34,426 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:35,430 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:36,429 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:37,424 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:38,426 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:38,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:39,428 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:40,433 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:41,447 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:42,449 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:43,447 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:43,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:44,452 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:45,455 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:46,444 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:47,451 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:48,450 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:48,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:49,451 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:50,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:51,464 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:52,470 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:53,453 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:53,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:54:54,461 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:54,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:55,465 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:56,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:57,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:58,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:59,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:59,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:00,483 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:01,476 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:02,484 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:03,475 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:04,482 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:04,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:05,479 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:06,487 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:07,492 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:08,488 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:09,503 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:09,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:10,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:11,501 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:12,492 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:13,495 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:14,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:14,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:15,494 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:16,494 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:17,496 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:18,501 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:19,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:19,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:20,518 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:21,527 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:22,522 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:23,513 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:23,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:55:24,510 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:25,516 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:25,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:26,516 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:27,521 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:28,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:29,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:30,538 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:30,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:31,528 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:32,533 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:33,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:34,530 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:35,526 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:35,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:36,536 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:37,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:38,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:39,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:40,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:40,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:41,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:42,535 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:43,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:44,549 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:45,545 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:45,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:46,548 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:47,556 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:48,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:49,556 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:50,549 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:50,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:51,557 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:52,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:53,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:55:53,574 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:54,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:55,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:56,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:56,582 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:57,576 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:58,578 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:59,584 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:00,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:01,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:01,582 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:02,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:03,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:04,585 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:05,597 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:06,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:06,595 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:07,590 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:08,593 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:09,594 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:10,595 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:11,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:11,613 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:12,601 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:13,618 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:14,618 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:15,616 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:16,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:16,606 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:17,621 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:18,620 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:19,622 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:20,628 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:21,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:21,634 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:22,629 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:23,566 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:56:23,634 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:24,638 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:25,642 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:26,641 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:27,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:27,642 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:28,659 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:29,657 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:30,644 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:31,641 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:32,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:32,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:33,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:34,654 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:35,651 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:36,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:37,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:37,651 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:38,663 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:39,665 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:40,659 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:41,658 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:42,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:42,658 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:43,684 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:44,674 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:45,663 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:46,669 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:47,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:47,676 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:48,681 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:49,691 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:50,690 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:51,694 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:52,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:52,697 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:53,566 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:56:53,707 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:54,696 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:55,695 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:56,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:57,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:58,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:58,722 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:59,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:00,710 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:01,709 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:02,714 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:03,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:03,741 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:04,740 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:05,739 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:06,739 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:07,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:08,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:08,742 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:09,749 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:10,748 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:11,742 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:12,763 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:13,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:13,746 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:14,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:15,748 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:16,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:17,752 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:18,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:18,749 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:19,754 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:20,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:21,761 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:22,772 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:23,567 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:57:23,763 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:24,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:24,775 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:25,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:26,760 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:27,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:28,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:29,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:29,781 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:30,778 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:31,771 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:32,777 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:33,774 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:34,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:34,767 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:35,770 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:36,778 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:37,777 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:38,780 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:39,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:39,781 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:40,788 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:41,785 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:42,795 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:43,794 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:44,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:44,800 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:45,800 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:46,799 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:47,808 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:48,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:49,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:49,807 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:50,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:51,811 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:52,825 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:53,567 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:57:53,805 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:54,811 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:55,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:55,808 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:56,806 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:57,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:58,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:59,816 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:00,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:00,814 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:01,818 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:02,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:03,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:04,828 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:05,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:05,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:06,823 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:07,828 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:08,825 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:09,831 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:10,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:10,831 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:11,839 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:12,849 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:13,851 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:14,846 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:15,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:15,842 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:16,851 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:17,849 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:18,860 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:19,854 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:20,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:20,850 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:21,854 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:22,855 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:23,568 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:58:23,865 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:24,869 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:25,870 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:26,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:26,871 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:27,871 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:28,874 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:29,875 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:30,879 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:31,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:31,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:32,887 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:33,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:34,891 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:35,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:36,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:36,890 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:37,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:38,892 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:39,898 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:40,909 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:41,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:41,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:42,901 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:43,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:44,907 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:45,910 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:46,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:46,913 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:47,915 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:48,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:49,914 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:50,912 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:51,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:51,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:52,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:53,568 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:58:53,915 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:54,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:55,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:56,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:57,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:57,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:58,944 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:59,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:00,954 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:01,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:02,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:02,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:03,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:04,954 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:05,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:06,971 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:07,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:07,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:08,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:09,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:10,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:11,982 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:12,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:12,982 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:13,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:14,985 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:15,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:16,999 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:17,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:18,007 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:19,003 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:19,995 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:21,000 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:22,007 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:22,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:23,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:23,569 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:59:24,009 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:25,023 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:26,026 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:27,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:28,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:28,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:29,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:30,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:31,034 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:32,044 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:33,038 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:33,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:34,032 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:35,055 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:36,050 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:37,054 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:38,048 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:38,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:39,053 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:40,048 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:41,062 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:42,063 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:43,064 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:43,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:44,075 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:45,073 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:46,072 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:47,075 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:48,092 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:48,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:49,078 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:50,080 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:51,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:52,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:53,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:53,569 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:59:54,089 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:54,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:55,095 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:56,102 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:57,120 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:58,107 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:59,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:59,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:00,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:01,116 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:02,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:03,113 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:04,136 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:04,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:05,117 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:06,122 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:07,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:08,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:09,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:09,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:10,124 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:11,125 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:12,134 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:13,135 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:14,133 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:14,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:15,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:16,137 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:17,143 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:18,150 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:19,148 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:19,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:20,149 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:21,152 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:22,163 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:23,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:23,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:00:24,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:25,166 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:25,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:26,175 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:27,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:28,172 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:29,171 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:30,169 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:30,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:31,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:32,175 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:33,179 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:34,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:35,190 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:35,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:36,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:37,204 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:38,182 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:39,194 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:40,194 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:40,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:41,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:42,192 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:43,183 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:44,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:45,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:45,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:46,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:47,190 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:48,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:49,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:50,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:50,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:51,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:52,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:53,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:53,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:00:54,224 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:55,212 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:56,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:56,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:57,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:58,209 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:59,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:00,215 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:01,215 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:01,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:02,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:03,219 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:04,223 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:05,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:06,223 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:06,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:07,225 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:08,233 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:09,235 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:10,237 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:11,243 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:11,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:12,244 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:13,245 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:14,244 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:15,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:16,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:16,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:17,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:18,264 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:19,264 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:20,266 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:21,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:21,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:22,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:23,271 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:23,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:01:24,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:25,267 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:26,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:27,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:27,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:28,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:29,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:30,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:30,766 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:31,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:32,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:32,686 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:33,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:34,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:35,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:36,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:37,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:37,686 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:38,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:39,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:40,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:41,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:42,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:42,687 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:43,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:44,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:45,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:46,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:47,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:47,688 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:48,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:49,271 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:49,889 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:01:49,889 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 21:01:49,890 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 with policy now +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:01:49,891 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:49,891 DEBUG SenderThread:1737239 [sender.py:send():379] send: telemetry +2024-12-31 21:01:49,891 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:01:49,973 DEBUG SenderThread:1737239 [sender.py:send():379] send: exit +2024-12-31 21:01:49,973 INFO SenderThread:1737239 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:01:49,973 INFO SenderThread:1737239 [sender.py:send_exit():588] handling runtime: 926 +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:send_exit():594] send defer +2024-12-31 21:01:49,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,974 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:01:49,974 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:01:49,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,974 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:01:49,975 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:49,975 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:01:49,975 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:01:49,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,975 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:01:49,975 INFO HandlerThread:1737239 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:01:49,975 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:01:49,975 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:01:49,976 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:01:49,976 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:01:50,028 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:01:50,029 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:01:50,029 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:50,029 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:01:50,029 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:01:50,029 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:50,029 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:01:50,169 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/qqawmlse-media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:50,773 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 21:01:51,039 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/h9z1qd6j-media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:55,416 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:55,416 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:55,417 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,417 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:01:55,417 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:01:55,417 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:01:55,417 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,418 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:01:55,418 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 21:01:55,418 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:01:55,419 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:55,419 DEBUG SenderThread:1737239 [sender.py:send():379] send: summary +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:55,423 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:01:55,424 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,424 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:01:55,424 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,424 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:01:55,427 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:55,553 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:01:55,554 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,554 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:01:55,554 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,554 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:01:55,776 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 21:01:55,777 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:01:55,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:01:55,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:58,233 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:01:58,233 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:01:58,233 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:58,234 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:01:58,234 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:58,234 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:01:58,234 INFO SenderThread:1737239 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:01:58,240 INFO SenderThread:1737239 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:01:58,254 INFO SenderThread:1737239 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:01:58,263 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:01:58,263 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:58,263 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:01:58,264 DEBUG SenderThread:1737239 [sender.py:send():379] send: artifact +2024-12-31 21:01:58,778 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:58,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:59,219 INFO wandb-upload_1:1737239 [upload_job.py:push():88] Uploaded file /tmp/tmpx0ryqqaf/wandb-job.json +2024-12-31 21:01:59,269 INFO wandb-upload_2:1737239 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsqvtcw6o +2024-12-31 21:01:59,273 INFO wandb-upload_0:1737239 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmzppz3e_ +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExMjAxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA3MTYzNw==', 'versionIndex': 388}}} +2024-12-31 21:01:59,790 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml config.yaml +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json wandb-summary.json +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt requirements.txt +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log output.log +2024-12-31 21:02:00,780 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch diff.patch +2024-12-31 21:02:00,784 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:02:00,786 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:00,787 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:00,787 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:02:00,787 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:00,787 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:02:00,787 INFO SenderThread:1737239 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:02:00,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:02:00,977 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 21:02:00,977 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:00,978 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt +2024-12-31 21:02:00,990 INFO wandb-upload_2:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:02:01,082 INFO wandb-upload_3:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:02:01,282 INFO Thread-11 (_thread_body):1737239 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:02:01,282 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,282 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:02:01,282 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,283 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:02:01,283 INFO SenderThread:1737239 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:02:01,284 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,284 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:02:01,284 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:02:01,457 INFO SenderThread:1737239 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:02:01,457 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:02:01,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,458 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:02:01,458 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,458 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:02:01,458 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:02:01,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,458 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send():379] send: final +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send():379] send: footer +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,459 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:02:01,460 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:02:01,460 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:01,461 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:02:01,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:02:01,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:02:01,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:02:01,472 INFO HandlerThread:1737239 [handler.py:finish():882] shutting down handler +2024-12-31 21:02:02,460 INFO WriterThread:1737239 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [sender.py:finish():1615] shutting down sender +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [file_pusher.py:join():175] waiting for file pusher diff --git a/kccehv5k/wandb/debug.log b/kccehv5k/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3771954035b92f652620f179459bd09a33fa9283 --- /dev/null +++ b/kccehv5k/wandb/debug.log @@ -0,0 +1,37 @@ +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Configure stats pid to 1737022 +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/settings +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug.log +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug-internal.log +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():569] calling init triggers +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k'} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():619] starting backend +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():623] setting up manager +2024-12-31 20:46:23,178 INFO MainThread:1737022 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 20:46:23,180 INFO MainThread:1737022 [wandb_init.py:init():631] backend started and connected +2024-12-31 20:46:23,196 INFO MainThread:1737022 [wandb_init.py:init():720] updated telemetry +2024-12-31 20:46:23,202 INFO MainThread:1737022 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 20:46:23,474 INFO MainThread:1737022 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 20:46:23,539 INFO MainThread:1737022 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 20:46:23,540 INFO MainThread:1737022 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 20:46:23,918 INFO MainThread:1737022 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 20:46:23,918 INFO MainThread:1737022 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/tb, True +2024-12-31 20:46:42,413 INFO MainThread:1737022 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/kccehv5k +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_restore():2352] restore +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_restore():2358] restore done +2024-12-31 21:02:02,473 INFO MainThread:1737022 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:02:02,473 INFO MainThread:1737022 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:02:02,478 INFO MainThread:1737022 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..31f33533b76f7428e7dfec30b7d0ed852d82e44c --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py @@ -0,0 +1,595 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + "config.grasped_obj_name": grasped_obj_name, + "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + total_num_episodes += filtered_run.summary_metrics['episode'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e45502a83b250cb754a83247a928c1301c584e4 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735695983 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 2 + - 14 + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c4ce27f9aa9d8063f3e928878d0c5414565da1d7 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch @@ -0,0 +1,66 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 3b7c93f..6c044fc 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -129,7 +129,12 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1051_0' # all + + # RGBD+mask+act history +-wandb_run_id: '1000_0' # all ++# wandb_run_id: '1000_0' # all ++ ++# all books, 10/20 demos per book ++# RGBD+mask+act history ++wandb_run_id: '1002_0' ++ + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +@@ -174,7 +179,7 @@ residual_weight: '${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name + + # old_experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${name_of_expert_demo}_valgroups_${val_num_groups}/${old_feature_type}/${seed}/${agent.name}/${wandb_run_id} + experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +-final_experiment_dir: ${experiment_dir}/${oc.generate_run_id} ++final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} + # ${now:%H%M%S} + hydra: + run: +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 5ed3eb4..f520cd4 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -148,7 +148,13 @@ remote_host = 'grasp-login1' + # ] + + run_id_list = [ +- '1000_0', ++ '1002_0', ++ '1009_1', ++ '1009_2', ++ '1022_0', ++ '1022_0', ++ '1023_0', ++ '1007_0', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 3482538..31f3353 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -471,11 +471,13 @@ class Workspace: + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ +- "jobType": "eval", ++ "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, +- "state": "finished" ++ "state": "finished", ++ "config.grasped_obj_name": grasped_obj_name, ++ "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64a854c20425dfef66eba6f37df29723879a9569 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c4871aa0a38b24cd6b714c4d872f1bdf15ee9e1368eca5c4c5cca7795db545 +size 1147449 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..478a8702384d36c3b932341e4bf3b9c980617bf0 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f99c821ca0f3dcf058d7bdc57f31c99fec024325049b678d22e60cdfcb7032ce +size 1154686 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1833c678ad3af1637f1656faaccc72b42325eea6 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497369a7fef0c32f3ee5bafccfc53b7a3889416759469d55aac8abeb7d3c5011 +size 1153780 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68a29cad76ffa17af095886eb7b17779cb1c8e8d --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8beb81bb83090b35f94094b007cb72a5ac5e21914596e2821215154f3398a6 +size 1163442 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea389243cba17f405864fd49527bfbc0e070f74f --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8469a9945cf9312a1f45cb2c0e3bef7b03b0c9263ac88eff33e692616036151 +size 1242957 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea389243cba17f405864fd49527bfbc0e070f74f --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8469a9945cf9312a1f45cb2c0e3bef7b03b0c9263ac88eff33e692616036151 +size 1242957 diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f67174d524842cd4f2126b8b591c0763f6b9a5aa --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log @@ -0,0 +1,6253 @@ +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735696002.414021]: resetting environment +[INFO] [1735696002.418974]: cleared current plan +[INFO] [1735696002.420590]: moving to home +[INFO] [1735696003.623604]: reached home +[INFO] [1735696003.624871]: reset action history +[INFO] [1735696006.384282]: environment reset +Starting episode 0 +[INFO] [1735696006.386743]: resetting environment +[INFO] [1735696006.389297]: cleared current plan +[INFO] [1735696006.390135]: moving to home +[INFO] [1735696007.392617]: reached home +[INFO] [1735696007.394014]: reset action history +[INFO] [1735696010.153112]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735696051.661548]: resetting environment +[INFO] [1735696051.665500]: cleared current plan +[INFO] [1735696051.666763]: moving to home +[INFO] [1735696052.668893]: reached home +[INFO] [1735696052.670074]: reset action history +[INFO] [1735696055.377456]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/episode_rosbags/episode_0_2024-12-31-20-47-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1737022.log +started roslaunch server http://158.130.50.37:43209/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1737565] +process[print_text-2]: started with pid [1737589] +process[rosbag_record-3]: started with pid [1737590] +started bagging! +[INFO] [1735696056.129211]: received mask_obs with header seq: 1 +stamp: + secs: 1735696056 + nsecs: 129437923 +frame_id: '' and grasped_object_mask header seq: 100768 +stamp: + secs: 1735696055 + nsecs: 910109758 +frame_id: "camera_color_optical_frame" +[INFO] [1735696056.318039]: received mask_obs with header seq: 2 +stamp: + secs: 1735696056 + nsecs: 318950891 +frame_id: '' and grasped_object_mask header seq: 100774 +stamp: + secs: 1735696056 + nsecs: 109952688 +frame_id: "camera_color_optical_frame" +[INFO] [1735696056.714803]: received mask_obs with header seq: 3 +stamp: + secs: 1735696056 + nsecs: 715666294 +frame_id: '' and grasped_object_mask header seq: 100786 +stamp: + secs: 1735696056 + nsecs: 509619951 +frame_id: "camera_color_optical_frame" +[INFO] [1735696056.895681]: received mask_obs with header seq: 4 +stamp: + secs: 1735696056 + nsecs: 894591331 +frame_id: '' and grasped_object_mask header seq: 100791 +stamp: + secs: 1735696056 + nsecs: 676148176 +frame_id: "camera_color_optical_frame" +[INFO] [1735696057.073069]: received mask_obs with header seq: 5 +stamp: + secs: 1735696057 + nsecs: 73992967 +frame_id: '' and grasped_object_mask header seq: 100796 +stamp: + secs: 1735696056 + nsecs: 842674971 +frame_id: "camera_color_optical_frame" +[INFO] [1735696057.257668]: received mask_obs with header seq: 6 +stamp: + secs: 1735696057 + nsecs: 258597373 +frame_id: '' and grasped_object_mask header seq: 100802 +stamp: + secs: 1735696057 + nsecs: 42507887 +frame_id: "camera_color_optical_frame" +[INFO] [1735696057.454191]: received mask_obs with header seq: 7 +stamp: + secs: 1735696057 + nsecs: 455110549 +frame_id: '' and grasped_object_mask header seq: 100807 +stamp: + secs: 1735696057 + nsecs: 209036589 +frame_id: "camera_color_optical_frame" +[INFO] [1735696057.651731]: received mask_obs with header seq: 8 +stamp: + secs: 1735696057 + nsecs: 652661323 +frame_id: '' and grasped_object_mask header seq: 100813 +stamp: + secs: 1735696057 + nsecs: 408871412 +frame_id: "camera_color_optical_frame" +[INFO] [1735696057.844424]: received mask_obs with header seq: 9 +stamp: + secs: 1735696057 + nsecs: 845223903 +frame_id: '' and grasped_object_mask header seq: 100819 +stamp: + secs: 1735696057 + nsecs: 608705759 +frame_id: "camera_color_optical_frame" +[INFO] [1735696058.043269]: received mask_obs with header seq: 10 +stamp: + secs: 1735696058 + nsecs: 43983221 +frame_id: '' and grasped_object_mask header seq: 100825 +stamp: + secs: 1735696057 + nsecs: 808540344 +frame_id: "camera_color_optical_frame" +[INFO] [1735696058.239326]: received mask_obs with header seq: 11 +stamp: + secs: 1735696058 + nsecs: 239871263 +frame_id: '' and grasped_object_mask header seq: 100831 +stamp: + secs: 1735696058 + nsecs: 8375406 +frame_id: "camera_color_optical_frame" +[INFO] [1735696058.438298]: received mask_obs with header seq: 12 +stamp: + secs: 1735696058 + nsecs: 439280748 +frame_id: '' and grasped_object_mask header seq: 100837 +stamp: + secs: 1735696058 + nsecs: 208211660 +frame_id: "camera_color_optical_frame" +[INFO] [1735696058.638040]: received mask_obs with header seq: 13 +stamp: + secs: 1735696058 + nsecs: 639005661 +frame_id: '' and grasped_object_mask header seq: 100843 +stamp: + secs: 1735696058 + nsecs: 408052444 +frame_id: "camera_color_optical_frame" +[INFO] [1735696058.832189]: received mask_obs with header seq: 14 +stamp: + secs: 1735696058 + nsecs: 833175182 +frame_id: '' and grasped_object_mask header seq: 100849 +stamp: + secs: 1735696058 + nsecs: 607893467 +frame_id: "camera_color_optical_frame" +[INFO] [1735696059.027599]: received mask_obs with header seq: 15 +stamp: + secs: 1735696059 + nsecs: 28589963 +frame_id: '' and grasped_object_mask header seq: 100854 +stamp: + secs: 1735696058 + nsecs: 774428368 +frame_id: "camera_color_optical_frame" +[INFO] [1735696059.218611]: received mask_obs with header seq: 16 +stamp: + secs: 1735696059 + nsecs: 219327211 +frame_id: '' and grasped_object_mask header seq: 100860 +stamp: + secs: 1735696058 + nsecs: 974269867 +frame_id: "camera_color_optical_frame" +[INFO] [1735696059.414217]: received mask_obs with header seq: 17 +stamp: + secs: 1735696059 + nsecs: 414196491 +frame_id: '' and grasped_object_mask header seq: 100866 +stamp: + secs: 1735696059 + nsecs: 174110413 +frame_id: "camera_color_optical_frame" +[INFO] [1735696059.609615]: received mask_obs with header seq: 18 +stamp: + secs: 1735696059 + nsecs: 610627174 +frame_id: '' and grasped_object_mask header seq: 100872 +stamp: + secs: 1735696059 + nsecs: 373944044 +frame_id: "camera_color_optical_frame" +[INFO] [1735696059.804354]: received mask_obs with header seq: 19 +stamp: + secs: 1735696059 + nsecs: 805207967 +frame_id: '' and grasped_object_mask header seq: 100878 +stamp: + secs: 1735696059 + nsecs: 573777914 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.006506]: received mask_obs with header seq: 20 +stamp: + secs: 1735696060 + nsecs: 7489204 +frame_id: '' and grasped_object_mask header seq: 100884 +stamp: + secs: 1735696059 + nsecs: 773612499 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.206224]: received mask_obs with header seq: 21 +stamp: + secs: 1735696060 + nsecs: 207175970 +frame_id: '' and grasped_object_mask header seq: 100890 +stamp: + secs: 1735696059 + nsecs: 973445892 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.403976]: received mask_obs with header seq: 22 +stamp: + secs: 1735696060 + nsecs: 403711795 +frame_id: '' and grasped_object_mask header seq: 100896 +stamp: + secs: 1735696060 + nsecs: 173279762 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.602570]: received mask_obs with header seq: 23 +stamp: + secs: 1735696060 + nsecs: 603459596 +frame_id: '' and grasped_object_mask header seq: 100902 +stamp: + secs: 1735696060 + nsecs: 373118877 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.798035]: received mask_obs with header seq: 24 +stamp: + secs: 1735696060 + nsecs: 798488378 +frame_id: '' and grasped_object_mask header seq: 100908 +stamp: + secs: 1735696060 + nsecs: 572957993 +frame_id: "camera_color_optical_frame" +[INFO] [1735696060.988506]: received mask_obs with header seq: 25 +stamp: + secs: 1735696060 + nsecs: 989491224 +frame_id: '' and grasped_object_mask header seq: 100913 +stamp: + secs: 1735696060 + nsecs: 739491224 +frame_id: "camera_color_optical_frame" +[INFO] [1735696061.182104]: received mask_obs with header seq: 26 +stamp: + secs: 1735696061 + nsecs: 183053970 +frame_id: '' and grasped_object_mask header seq: 100919 +stamp: + secs: 1735696060 + nsecs: 939330339 +frame_id: "camera_color_optical_frame" +[INFO] [1735696061.386562]: received mask_obs with header seq: 27 +stamp: + secs: 1735696061 + nsecs: 387248277 +frame_id: '' and grasped_object_mask header seq: 100925 +stamp: + secs: 1735696061 + nsecs: 139169455 +frame_id: "camera_color_optical_frame" +[INFO] [1735696061.585590]: received mask_obs with header seq: 28 +stamp: + secs: 1735696061 + nsecs: 586542844 +frame_id: '' and grasped_object_mask header seq: 100931 +stamp: + secs: 1735696061 + nsecs: 339007378 +frame_id: "camera_color_optical_frame" +[INFO] [1735696061.794268]: received mask_obs with header seq: 29 +stamp: + secs: 1735696061 + nsecs: 793956756 +frame_id: '' and grasped_object_mask header seq: 100937 +stamp: + secs: 1735696061 + nsecs: 538845301 +frame_id: "camera_color_optical_frame" +[INFO] [1735696061.996348]: received mask_obs with header seq: 30 +stamp: + secs: 1735696061 + nsecs: 996429681 +frame_id: '' and grasped_object_mask header seq: 100943 +stamp: + secs: 1735696061 + nsecs: 738683939 +frame_id: "camera_color_optical_frame" +[INFO] [1735696062.188424]: received mask_obs with header seq: 31 +stamp: + secs: 1735696062 + nsecs: 189279556 +frame_id: '' and grasped_object_mask header seq: 100949 +stamp: + secs: 1735696061 + nsecs: 938521862 +frame_id: "camera_color_optical_frame" +[INFO] [1735696062.379840]: received mask_obs with header seq: 32 +stamp: + secs: 1735696062 + nsecs: 380802869 +frame_id: '' and grasped_object_mask header seq: 100955 +stamp: + secs: 1735696062 + nsecs: 138359785 +frame_id: "camera_color_optical_frame" +[INFO] [1735696062.579416]: received mask_obs with header seq: 33 +stamp: + secs: 1735696062 + nsecs: 580303668 +frame_id: '' and grasped_object_mask header seq: 100961 +stamp: + secs: 1735696062 + nsecs: 339962006 +frame_id: "camera_color_optical_frame" +[INFO] [1735696062.774326]: received mask_obs with header seq: 34 +stamp: + secs: 1735696062 + nsecs: 774638891 +frame_id: '' and grasped_object_mask header seq: 100967 +stamp: + secs: 1735696062 + nsecs: 542131186 +frame_id: "camera_color_optical_frame" +[INFO] [1735696062.968749]: received mask_obs with header seq: 35 +stamp: + secs: 1735696062 + nsecs: 969617366 +frame_id: '' and grasped_object_mask header seq: 100973 +stamp: + secs: 1735696062 + nsecs: 744386435 +frame_id: "camera_color_optical_frame" +[INFO] [1735696063.168984]: received mask_obs with header seq: 36 +stamp: + secs: 1735696063 + nsecs: 169937372 +frame_id: '' and grasped_object_mask header seq: 100979 +stamp: + secs: 1735696062 + nsecs: 946728468 +frame_id: "camera_color_optical_frame" +[INFO] [1735696063.367965]: received mask_obs with header seq: 37 +stamp: + secs: 1735696063 + nsecs: 368749380 +frame_id: '' and grasped_object_mask header seq: 100985 +stamp: + secs: 1735696063 + nsecs: 149155617 +frame_id: "camera_color_optical_frame" +[INFO] [1735696063.572505]: received mask_obs with header seq: 38 +stamp: + secs: 1735696063 + nsecs: 573458433 +frame_id: '' and grasped_object_mask header seq: 100990 +stamp: + secs: 1735696063 + nsecs: 316141367 +frame_id: "camera_color_optical_frame" +[INFO] [1735696063.768343]: received mask_obs with header seq: 39 +stamp: + secs: 1735696063 + nsecs: 769304037 +frame_id: '' and grasped_object_mask header seq: 100997 +stamp: + secs: 1735696063 + nsecs: 549032450 +frame_id: "camera_color_optical_frame" +[INFO] [1735696063.966173]: received mask_obs with header seq: 40 +stamp: + secs: 1735696063 + nsecs: 967003345 +frame_id: '' and grasped_object_mask header seq: 101003 +stamp: + secs: 1735696063 + nsecs: 748641014 +frame_id: "camera_color_optical_frame" +[INFO] [1735696064.170793]: received mask_obs with header seq: 41 +stamp: + secs: 1735696064 + nsecs: 171794652 +frame_id: '' and grasped_object_mask header seq: 101009 +stamp: + secs: 1735696063 + nsecs: 948238611 +frame_id: "camera_color_optical_frame" +[INFO] [1735696064.368955]: received mask_obs with header seq: 42 +stamp: + secs: 1735696064 + nsecs: 369565248 +frame_id: '' and grasped_object_mask header seq: 101015 +stamp: + secs: 1735696064 + nsecs: 147825718 +frame_id: "camera_color_optical_frame" +[INFO] [1735696064.572686]: received mask_obs with header seq: 43 +stamp: + secs: 1735696064 + nsecs: 573410272 +frame_id: '' and grasped_object_mask header seq: 101021 +stamp: + secs: 1735696064 + nsecs: 347434759 +frame_id: "camera_color_optical_frame" +[INFO] [1735696064.768800]: received mask_obs with header seq: 44 +stamp: + secs: 1735696064 + nsecs: 769537448 +frame_id: '' and grasped_object_mask header seq: 101027 +stamp: + secs: 1735696064 + nsecs: 547041893 +frame_id: "camera_color_optical_frame" +[INFO] [1735696064.966241]: received mask_obs with header seq: 45 +stamp: + secs: 1735696064 + nsecs: 966795682 +frame_id: '' and grasped_object_mask header seq: 101033 +stamp: + secs: 1735696064 + nsecs: 746637821 +frame_id: "camera_color_optical_frame" +[INFO] [1735696065.154977]: received mask_obs with header seq: 46 +stamp: + secs: 1735696065 + nsecs: 155863285 +frame_id: '' and grasped_object_mask header seq: 101038 +stamp: + secs: 1735696064 + nsecs: 912959576 +frame_id: "camera_color_optical_frame" +[INFO] [1735696065.346093]: received mask_obs with header seq: 47 +stamp: + secs: 1735696065 + nsecs: 346676111 +frame_id: '' and grasped_object_mask header seq: 101044 +stamp: + secs: 1735696065 + nsecs: 112534285 +frame_id: "camera_color_optical_frame" +[INFO] [1735696065.543745]: received mask_obs with header seq: 48 +stamp: + secs: 1735696065 + nsecs: 544661521 +frame_id: '' and grasped_object_mask header seq: 101050 +stamp: + secs: 1735696065 + nsecs: 312133789 +frame_id: "camera_color_optical_frame" +[INFO] [1735696065.740358]: received mask_obs with header seq: 49 +stamp: + secs: 1735696065 + nsecs: 741373777 +frame_id: '' and grasped_object_mask header seq: 101056 +stamp: + secs: 1735696065 + nsecs: 511745453 +frame_id: "camera_color_optical_frame" +[INFO] [1735696065.936207]: received mask_obs with header seq: 50 +stamp: + secs: 1735696065 + nsecs: 936383008 +frame_id: '' and grasped_object_mask header seq: 101062 +stamp: + secs: 1735696065 + nsecs: 711345196 +frame_id: "camera_color_optical_frame" +[INFO] [1735696066.136972]: received mask_obs with header seq: 51 +stamp: + secs: 1735696066 + nsecs: 137947320 +frame_id: '' and grasped_object_mask header seq: 101068 +stamp: + secs: 1735696065 + nsecs: 910933733 +frame_id: "camera_color_optical_frame" +[INFO] [1735696066.336866]: received mask_obs with header seq: 52 +stamp: + secs: 1735696066 + nsecs: 337891101 +frame_id: '' and grasped_object_mask header seq: 101074 +stamp: + secs: 1735696066 + nsecs: 110512018 +frame_id: "camera_color_optical_frame" +[INFO] [1735696066.536620]: received mask_obs with header seq: 53 +stamp: + secs: 1735696066 + nsecs: 537290334 +frame_id: '' and grasped_object_mask header seq: 101080 +stamp: + secs: 1735696066 + nsecs: 310110807 +frame_id: "camera_color_optical_frame" +[INFO] [1735696066.735349]: received mask_obs with header seq: 54 +stamp: + secs: 1735696066 + nsecs: 736129045 +frame_id: '' and grasped_object_mask header seq: 101086 +stamp: + secs: 1735696066 + nsecs: 509719133 +frame_id: "camera_color_optical_frame" +[INFO] [1735696066.936621]: received mask_obs with header seq: 55 +stamp: + secs: 1735696066 + nsecs: 937610387 +frame_id: '' and grasped_object_mask header seq: 101092 +stamp: + secs: 1735696066 + nsecs: 709316492 +frame_id: "camera_color_optical_frame" +[INFO] [1735696067.138296]: received mask_obs with header seq: 56 +stamp: + secs: 1735696067 + nsecs: 139167070 +frame_id: '' and grasped_object_mask header seq: 101098 +stamp: + secs: 1735696066 + nsecs: 908901930 +frame_id: "camera_color_optical_frame" +[INFO] [1735696067.334396]: received mask_obs with header seq: 57 +stamp: + secs: 1735696067 + nsecs: 334457159 +frame_id: '' and grasped_object_mask header seq: 101104 +stamp: + secs: 1735696067 + nsecs: 108475924 +frame_id: "camera_color_optical_frame" +[INFO] [1735696067.537869]: received mask_obs with header seq: 58 +stamp: + secs: 1735696067 + nsecs: 538886308 +frame_id: '' and grasped_object_mask header seq: 101110 +stamp: + secs: 1735696067 + nsecs: 308075666 +frame_id: "camera_color_optical_frame" +[INFO] [1735696067.735542]: received mask_obs with header seq: 59 +stamp: + secs: 1735696067 + nsecs: 736580371 +frame_id: '' and grasped_object_mask header seq: 101116 +stamp: + secs: 1735696067 + nsecs: 507687092 +frame_id: "camera_color_optical_frame" +[INFO] [1735696067.934534]: received mask_obs with header seq: 60 +stamp: + secs: 1735696067 + nsecs: 935564994 +frame_id: '' and grasped_object_mask header seq: 101122 +stamp: + secs: 1735696067 + nsecs: 707286835 +frame_id: "camera_color_optical_frame" +[INFO] [1735696068.139478]: received mask_obs with header seq: 61 +stamp: + secs: 1735696068 + nsecs: 140383481 +frame_id: '' and grasped_object_mask header seq: 101128 +stamp: + secs: 1735696067 + nsecs: 906875134 +frame_id: "camera_color_optical_frame" +[INFO] [1735696068.337277]: received mask_obs with header seq: 62 +stamp: + secs: 1735696068 + nsecs: 338073015 +frame_id: '' and grasped_object_mask header seq: 101134 +stamp: + secs: 1735696068 + nsecs: 106452227 +frame_id: "camera_color_optical_frame" +[INFO] [1735696068.536162]: received mask_obs with header seq: 63 +stamp: + secs: 1735696068 + nsecs: 536948680 +frame_id: '' and grasped_object_mask header seq: 101140 +stamp: + secs: 1735696068 + nsecs: 306051493 +frame_id: "camera_color_optical_frame" +[INFO] [1735696068.735431]: received mask_obs with header seq: 64 +stamp: + secs: 1735696068 + nsecs: 736493349 +frame_id: '' and grasped_object_mask header seq: 101146 +stamp: + secs: 1735696068 + nsecs: 505663157 +frame_id: "camera_color_optical_frame" +[INFO] [1735696068.935463]: received mask_obs with header seq: 65 +stamp: + secs: 1735696068 + nsecs: 936469554 +frame_id: '' and grasped_object_mask header seq: 101152 +stamp: + secs: 1735696068 + nsecs: 705261946 +frame_id: "camera_color_optical_frame" +[INFO] [1735696069.138780]: received mask_obs with header seq: 66 +stamp: + secs: 1735696069 + nsecs: 139838933 +frame_id: '' and grasped_object_mask header seq: 101158 +stamp: + secs: 1735696068 + nsecs: 904849291 +frame_id: "camera_color_optical_frame" +[INFO] [1735696069.336645]: received mask_obs with header seq: 67 +stamp: + secs: 1735696069 + nsecs: 337721347 +frame_id: '' and grasped_object_mask header seq: 101164 +stamp: + secs: 1735696069 + nsecs: 104425192 +frame_id: "camera_color_optical_frame" +[INFO] [1735696069.535534]: received mask_obs with header seq: 68 +stamp: + secs: 1735696069 + nsecs: 536610126 +frame_id: '' and grasped_object_mask header seq: 101170 +stamp: + secs: 1735696069 + nsecs: 304045200 +frame_id: "camera_color_optical_frame" +[INFO] [1735696069.734891]: received mask_obs with header seq: 69 +stamp: + secs: 1735696069 + nsecs: 735701799 +frame_id: '' and grasped_object_mask header seq: 101176 +stamp: + secs: 1735696069 + nsecs: 503661156 +frame_id: "camera_color_optical_frame" +[INFO] [1735696069.938560]: received mask_obs with header seq: 70 +stamp: + secs: 1735696069 + nsecs: 939594745 +frame_id: '' and grasped_object_mask header seq: 101182 +stamp: + secs: 1735696069 + nsecs: 703267097 +frame_id: "camera_color_optical_frame" +[INFO] [1735696070.144967]: received mask_obs with header seq: 71 +stamp: + secs: 1735696070 + nsecs: 145900964 +frame_id: '' and grasped_object_mask header seq: 101188 +stamp: + secs: 1735696069 + nsecs: 902860641 +frame_id: "camera_color_optical_frame" +[INFO] [1735696070.343211]: received mask_obs with header seq: 72 +stamp: + secs: 1735696070 + nsecs: 343887805 +frame_id: '' and grasped_object_mask header seq: 101194 +stamp: + secs: 1735696070 + nsecs: 102443218 +frame_id: "camera_color_optical_frame" +[INFO] [1735696070.538880]: received mask_obs with header seq: 73 +stamp: + secs: 1735696070 + nsecs: 539811134 +frame_id: '' and grasped_object_mask header seq: 101200 +stamp: + secs: 1735696070 + nsecs: 302036524 +frame_id: "camera_color_optical_frame" +[INFO] [1735696070.734385]: received mask_obs with header seq: 74 +stamp: + secs: 1735696070 + nsecs: 734971046 +frame_id: '' and grasped_object_mask header seq: 101206 +stamp: + secs: 1735696070 + nsecs: 501641035 +frame_id: "camera_color_optical_frame" +[INFO] [1735696070.935363]: received mask_obs with header seq: 75 +stamp: + secs: 1735696070 + nsecs: 936399936 +frame_id: '' and grasped_object_mask header seq: 101212 +stamp: + secs: 1735696070 + nsecs: 701233625 +frame_id: "camera_color_optical_frame" +[INFO] [1735696071.137001]: received mask_obs with header seq: 76 +stamp: + secs: 1735696071 + nsecs: 137866735 +frame_id: '' and grasped_object_mask header seq: 101218 +stamp: + secs: 1735696070 + nsecs: 900815487 +frame_id: "camera_color_optical_frame" +[INFO] [1735696071.334612]: received mask_obs with header seq: 77 +stamp: + secs: 1735696071 + nsecs: 335649251 +frame_id: '' and grasped_object_mask header seq: 101224 +stamp: + secs: 1735696071 + nsecs: 100384474 +frame_id: "camera_color_optical_frame" +[INFO] [1735696071.537301]: received mask_obs with header seq: 78 +stamp: + secs: 1735696071 + nsecs: 538194656 +frame_id: '' and grasped_object_mask header seq: 101230 +stamp: + secs: 1735696071 + nsecs: 299976349 +frame_id: "camera_color_optical_frame" +[INFO] [1735696071.737328]: received mask_obs with header seq: 79 +stamp: + secs: 1735696071 + nsecs: 738257646 +frame_id: '' and grasped_object_mask header seq: 101236 +stamp: + secs: 1735696071 + nsecs: 499591589 +frame_id: "camera_color_optical_frame" +[INFO] [1735696071.934898]: received mask_obs with header seq: 80 +stamp: + secs: 1735696071 + nsecs: 935892105 +frame_id: '' and grasped_object_mask header seq: 101242 +stamp: + secs: 1735696071 + nsecs: 699195385 +frame_id: "camera_color_optical_frame" +[INFO] [1735696072.137756]: received mask_obs with header seq: 81 +stamp: + secs: 1735696072 + nsecs: 138833045 +frame_id: '' and grasped_object_mask header seq: 101248 +stamp: + secs: 1735696071 + nsecs: 898787975 +frame_id: "camera_color_optical_frame" +[INFO] [1735696072.345349]: received mask_obs with header seq: 82 +stamp: + secs: 1735696072 + nsecs: 346067905 +frame_id: '' and grasped_object_mask header seq: 101254 +stamp: + secs: 1735696072 + nsecs: 98370314 +frame_id: "camera_color_optical_frame" +[INFO] [1735696072.541685]: received mask_obs with header seq: 83 +stamp: + secs: 1735696072 + nsecs: 542237997 +frame_id: '' and grasped_object_mask header seq: 101260 +stamp: + secs: 1735696072 + nsecs: 297969103 +frame_id: "camera_color_optical_frame" +[INFO] [1735696072.737490]: received mask_obs with header seq: 84 +stamp: + secs: 1735696072 + nsecs: 738510370 +frame_id: '' and grasped_object_mask header seq: 101266 +stamp: + secs: 1735696072 + nsecs: 497584581 +frame_id: "camera_color_optical_frame" +[INFO] [1735696072.937344]: received mask_obs with header seq: 85 +stamp: + secs: 1735696072 + nsecs: 938441753 +frame_id: '' and grasped_object_mask header seq: 101272 +stamp: + secs: 1735696072 + nsecs: 697188616 +frame_id: "camera_color_optical_frame" +[INFO] [1735696073.136403]: received mask_obs with header seq: 86 +stamp: + secs: 1735696073 + nsecs: 137336730 +frame_id: '' and grasped_object_mask header seq: 101278 +stamp: + secs: 1735696072 + nsecs: 896781683 +frame_id: "camera_color_optical_frame" +[INFO] [1735696073.334556]: received mask_obs with header seq: 87 +stamp: + secs: 1735696073 + nsecs: 335657119 +frame_id: '' and grasped_object_mask header seq: 101284 +stamp: + secs: 1735696073 + nsecs: 96363068 +frame_id: "camera_color_optical_frame" +[INFO] [1735696073.536306]: received mask_obs with header seq: 88 +stamp: + secs: 1735696073 + nsecs: 537353515 +frame_id: '' and grasped_object_mask header seq: 101290 +stamp: + secs: 1735696073 + nsecs: 295959711 +frame_id: "camera_color_optical_frame" +[INFO] [1735696073.734123]: received mask_obs with header seq: 89 +stamp: + secs: 1735696073 + nsecs: 735189914 +frame_id: '' and grasped_object_mask header seq: 101296 +stamp: + secs: 1735696073 + nsecs: 495572567 +frame_id: "camera_color_optical_frame" +[INFO] [1735696073.932576]: received mask_obs with header seq: 90 +stamp: + secs: 1735696073 + nsecs: 933527946 +frame_id: '' and grasped_object_mask header seq: 101302 +stamp: + secs: 1735696073 + nsecs: 695172548 +frame_id: "camera_color_optical_frame" +[INFO] [1735696074.138102]: received mask_obs with header seq: 91 +stamp: + secs: 1735696074 + nsecs: 139194250 +frame_id: '' and grasped_object_mask header seq: 101308 +stamp: + secs: 1735696073 + nsecs: 894761086 +frame_id: "camera_color_optical_frame" +[INFO] [1735696074.335187]: received mask_obs with header seq: 92 +stamp: + secs: 1735696074 + nsecs: 335816383 +frame_id: '' and grasped_object_mask header seq: 101314 +stamp: + secs: 1735696074 + nsecs: 94338179 +frame_id: "camera_color_optical_frame" +[INFO] [1735696074.534667]: received mask_obs with header seq: 93 +stamp: + secs: 1735696074 + nsecs: 535753965 +frame_id: '' and grasped_object_mask header seq: 101319 +stamp: + secs: 1735696074 + nsecs: 260662317 +frame_id: "camera_color_optical_frame" +[INFO] [1735696074.736604]: received mask_obs with header seq: 94 +stamp: + secs: 1735696074 + nsecs: 737701654 +frame_id: '' and grasped_object_mask header seq: 101326 +stamp: + secs: 1735696074 + nsecs: 493541002 +frame_id: "camera_color_optical_frame" +[INFO] [1735696074.935425]: received mask_obs with header seq: 95 +stamp: + secs: 1735696074 + nsecs: 936541080 +frame_id: '' and grasped_object_mask header seq: 101332 +stamp: + secs: 1735696074 + nsecs: 693139791 +frame_id: "camera_color_optical_frame" +[INFO] [1735696075.136390]: received mask_obs with header seq: 96 +stamp: + secs: 1735696075 + nsecs: 137457132 +frame_id: '' and grasped_object_mask header seq: 101338 +stamp: + secs: 1735696074 + nsecs: 892726421 +frame_id: "camera_color_optical_frame" +[INFO] [1735696075.336323]: received mask_obs with header seq: 97 +stamp: + secs: 1735696075 + nsecs: 337296724 +frame_id: '' and grasped_object_mask header seq: 101344 +stamp: + secs: 1735696075 + nsecs: 92301130 +frame_id: "camera_color_optical_frame" +[INFO] [1735696075.536147]: received mask_obs with header seq: 98 +stamp: + secs: 1735696075 + nsecs: 536934375 +frame_id: '' and grasped_object_mask header seq: 101350 +stamp: + secs: 1735696075 + nsecs: 291894197 +frame_id: "camera_color_optical_frame" +[INFO] [1735696075.734550]: received mask_obs with header seq: 99 +stamp: + secs: 1735696075 + nsecs: 735334157 +frame_id: '' and grasped_object_mask header seq: 101356 +stamp: + secs: 1735696075 + nsecs: 491506100 +frame_id: "camera_color_optical_frame" +[INFO] [1735696075.936484]: received mask_obs with header seq: 100 +stamp: + secs: 1735696075 + nsecs: 937589645 +frame_id: '' and grasped_object_mask header seq: 101362 +stamp: + secs: 1735696075 + nsecs: 691105843 +frame_id: "camera_color_optical_frame" +[INFO] [1735696076.136704]: received mask_obs with header seq: 101 +stamp: + secs: 1735696076 + nsecs: 137834310 +frame_id: '' and grasped_object_mask header seq: 101368 +stamp: + secs: 1735696075 + nsecs: 890695333 +frame_id: "camera_color_optical_frame" +[INFO] [1735696076.335610]: received mask_obs with header seq: 102 +stamp: + secs: 1735696076 + nsecs: 336424589 +frame_id: '' and grasped_object_mask header seq: 101374 +stamp: + secs: 1735696076 + nsecs: 90271950 +frame_id: "camera_color_optical_frame" +[INFO] [1735696076.537080]: received mask_obs with header seq: 103 +stamp: + secs: 1735696076 + nsecs: 537864685 +frame_id: '' and grasped_object_mask header seq: 101380 +stamp: + secs: 1735696076 + nsecs: 289866209 +frame_id: "camera_color_optical_frame" +[INFO] [1735696076.734181]: received mask_obs with header seq: 104 +stamp: + secs: 1735696076 + nsecs: 735014915 +frame_id: '' and grasped_object_mask header seq: 101386 +stamp: + secs: 1735696076 + nsecs: 489477873 +frame_id: "camera_color_optical_frame" +[INFO] [1735696076.937067]: received mask_obs with header seq: 105 +stamp: + secs: 1735696076 + nsecs: 938169002 +frame_id: '' and grasped_object_mask header seq: 101392 +stamp: + secs: 1735696076 + nsecs: 689077854 +frame_id: "camera_color_optical_frame" +[INFO] [1735696077.139501]: received mask_obs with header seq: 106 +stamp: + secs: 1735696077 + nsecs: 140463352 +frame_id: '' and grasped_object_mask header seq: 101398 +stamp: + secs: 1735696076 + nsecs: 888666391 +frame_id: "camera_color_optical_frame" +[INFO] [1735696077.334150]: received mask_obs with header seq: 107 +stamp: + secs: 1735696077 + nsecs: 335138559 +frame_id: '' and grasped_object_mask header seq: 101404 +stamp: + secs: 1735696077 + nsecs: 88244677 +frame_id: "camera_color_optical_frame" +[INFO] [1735696077.537102]: received mask_obs with header seq: 108 +stamp: + secs: 1735696077 + nsecs: 537991285 +frame_id: '' and grasped_object_mask header seq: 101410 +stamp: + secs: 1735696077 + nsecs: 287836313 +frame_id: "camera_color_optical_frame" +[INFO] [1735696077.736287]: received mask_obs with header seq: 109 +stamp: + secs: 1735696077 + nsecs: 737111091 +frame_id: '' and grasped_object_mask header seq: 101416 +stamp: + secs: 1735696077 + nsecs: 487442732 +frame_id: "camera_color_optical_frame" +[INFO] [1735696077.942371]: received mask_obs with header seq: 110 +stamp: + secs: 1735696077 + nsecs: 941989183 +frame_id: '' and grasped_object_mask header seq: 101422 +stamp: + secs: 1735696077 + nsecs: 687036991 +frame_id: "camera_color_optical_frame" +[INFO] [1735696078.150031]: received mask_obs with header seq: 111 +stamp: + secs: 1735696078 + nsecs: 150966644 +frame_id: '' and grasped_object_mask header seq: 101428 +stamp: + secs: 1735696077 + nsecs: 886619568 +frame_id: "camera_color_optical_frame" +[INFO] [1735696078.355736]: received mask_obs with header seq: 112 +stamp: + secs: 1735696078 + nsecs: 356565952 +frame_id: '' and grasped_object_mask header seq: 101435 +stamp: + secs: 1735696078 + nsecs: 119451523 +frame_id: "camera_color_optical_frame" +[INFO] [1735696078.551826]: received mask_obs with header seq: 113 +stamp: + secs: 1735696078 + nsecs: 552802562 +frame_id: '' and grasped_object_mask header seq: 101441 +stamp: + secs: 1735696078 + nsecs: 319882631 +frame_id: "camera_color_optical_frame" +[INFO] [1735696078.750223]: received mask_obs with header seq: 114 +stamp: + secs: 1735696078 + nsecs: 751148700 +frame_id: '' and grasped_object_mask header seq: 101447 +stamp: + secs: 1735696078 + nsecs: 520820618 +frame_id: "camera_color_optical_frame" +[INFO] [1735696078.948626]: received mask_obs with header seq: 115 +stamp: + secs: 1735696078 + nsecs: 949645519 +frame_id: '' and grasped_object_mask header seq: 101453 +stamp: + secs: 1735696078 + nsecs: 721844435 +frame_id: "camera_color_optical_frame" +[INFO] [1735696079.140805]: received mask_obs with header seq: 116 +stamp: + secs: 1735696079 + nsecs: 141905307 +frame_id: '' and grasped_object_mask header seq: 101458 +stamp: + secs: 1735696078 + nsecs: 889431000 +frame_id: "camera_color_optical_frame" +[INFO] [1735696079.340722]: received mask_obs with header seq: 117 +stamp: + secs: 1735696079 + nsecs: 340106010 +frame_id: '' and grasped_object_mask header seq: 101464 +stamp: + secs: 1735696079 + nsecs: 90612650 +frame_id: "camera_color_optical_frame" +[INFO] [1735696079.540156]: received mask_obs with header seq: 118 +stamp: + secs: 1735696079 + nsecs: 540516853 +frame_id: '' and grasped_object_mask header seq: 101471 +stamp: + secs: 1735696079 + nsecs: 324461937 +frame_id: "camera_color_optical_frame" +[INFO] [1735696079.744201]: received mask_obs with header seq: 119 +stamp: + secs: 1735696079 + nsecs: 745281696 +frame_id: '' and grasped_object_mask header seq: 101477 +stamp: + secs: 1735696079 + nsecs: 524300575 +frame_id: "camera_color_optical_frame" +[INFO] [1735696079.946090]: received mask_obs with header seq: 120 +stamp: + secs: 1735696079 + nsecs: 946752548 +frame_id: '' and grasped_object_mask header seq: 101483 +stamp: + secs: 1735696079 + nsecs: 724139452 +frame_id: "camera_color_optical_frame" +[INFO] [1735696080.147712]: received mask_obs with header seq: 121 +stamp: + secs: 1735696080 + nsecs: 148774862 +frame_id: '' and grasped_object_mask header seq: 101488 +stamp: + secs: 1735696079 + nsecs: 890672684 +frame_id: "camera_color_optical_frame" +[INFO] [1735696080.355733]: received mask_obs with header seq: 122 +stamp: + secs: 1735696080 + nsecs: 356725931 +frame_id: '' and grasped_object_mask header seq: 101495 +stamp: + secs: 1735696080 + nsecs: 123819113 +frame_id: "camera_color_optical_frame" +[INFO] [1735696080.551258]: received mask_obs with header seq: 123 +stamp: + secs: 1735696080 + nsecs: 552361488 +frame_id: '' and grasped_object_mask header seq: 101501 +stamp: + secs: 1735696080 + nsecs: 323658466 +frame_id: "camera_color_optical_frame" +[INFO] [1735696080.749680]: received mask_obs with header seq: 124 +stamp: + secs: 1735696080 + nsecs: 749973773 +frame_id: '' and grasped_object_mask header seq: 101507 +stamp: + secs: 1735696080 + nsecs: 523498297 +frame_id: "camera_color_optical_frame" +[INFO] [1735696080.945832]: received mask_obs with header seq: 125 +stamp: + secs: 1735696080 + nsecs: 946792364 +frame_id: '' and grasped_object_mask header seq: 101513 +stamp: + secs: 1735696080 + nsecs: 723338127 +frame_id: "camera_color_optical_frame" +[INFO] [1735696081.146140]: received mask_obs with header seq: 126 +stamp: + secs: 1735696081 + nsecs: 147260904 +frame_id: '' and grasped_object_mask header seq: 101519 +stamp: + secs: 1735696080 + nsecs: 923178196 +frame_id: "camera_color_optical_frame" +[INFO] [1735696081.346364]: received mask_obs with header seq: 127 +stamp: + secs: 1735696081 + nsecs: 347171306 +frame_id: '' and grasped_object_mask header seq: 101525 +stamp: + secs: 1735696081 + nsecs: 123018026 +frame_id: "camera_color_optical_frame" +[INFO] [1735696081.542824]: received mask_obs with header seq: 128 +stamp: + secs: 1735696081 + nsecs: 543695211 +frame_id: '' and grasped_object_mask header seq: 101531 +stamp: + secs: 1735696081 + nsecs: 322854757 +frame_id: "camera_color_optical_frame" +[INFO] [1735696081.743055]: received mask_obs with header seq: 129 +stamp: + secs: 1735696081 + nsecs: 744113922 +frame_id: '' and grasped_object_mask header seq: 101537 +stamp: + secs: 1735696081 + nsecs: 522689342 +frame_id: "camera_color_optical_frame" +[INFO] [1735696081.943195]: received mask_obs with header seq: 130 +stamp: + secs: 1735696081 + nsecs: 943413496 +frame_id: '' and grasped_object_mask header seq: 101543 +stamp: + secs: 1735696081 + nsecs: 722522974 +frame_id: "camera_color_optical_frame" +[INFO] [1735696082.144885]: received mask_obs with header seq: 131 +stamp: + secs: 1735696082 + nsecs: 144985675 +frame_id: '' and grasped_object_mask header seq: 101549 +stamp: + secs: 1735696081 + nsecs: 922356367 +frame_id: "camera_color_optical_frame" +[INFO] [1735696082.342335]: received mask_obs with header seq: 132 +stamp: + secs: 1735696082 + nsecs: 343254327 +frame_id: '' and grasped_object_mask header seq: 101555 +stamp: + secs: 1735696082 + nsecs: 122189760 +frame_id: "camera_color_optical_frame" +[INFO] [1735696082.380134]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735696082.544325]: received mask_obs with header seq: 133 +stamp: + secs: 1735696082 + nsecs: 545401811 +frame_id: '' and grasped_object_mask header seq: 101561 +stamp: + secs: 1735696082 + nsecs: 322028399 +frame_id: "camera_color_optical_frame" +[INFO] [1735696083.123678]: Stored episode 1. +[INFO] [1735696083.124034]: Saving video... +[INFO] [1735696083.308458]: Video saved! +Starting episode 1 +[INFO] [1735696083.322447]: resetting environment +[INFO] [1735696083.326954]: cleared current plan +[INFO] [1735696083.327143]: moving to home +[INFO] [1735696086.929352]: reached home +[INFO] [1735696086.929812]: reset action history +[INFO] [1735696089.641727]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735696091.439378]: resetting environment +[INFO] [1735696091.456519]: cleared current plan +[INFO] [1735696091.458436]: moving to home +[INFO] [1735696092.463675]: reached home +[INFO] [1735696092.466400]: reset action history +[INFO] [1735696095.174684]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/episode_rosbags/episode_1_2024-12-31-20-48-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1737022.log +started roslaunch server http://158.130.50.37:44245/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1737701] +process[print_text-5]: started with pid [1737702] +process[rosbag_record-6]: started with pid [1737703] +started bagging! +[INFO] [1735696095.867018]: received mask_obs with header seq: 134 +stamp: + secs: 1735696095 + nsecs: 867152929 +frame_id: '' and grasped_object_mask header seq: 101962 +stamp: + secs: 1735696095 + nsecs: 678086281 +frame_id: "camera_color_optical_frame" +[INFO] [1735696096.082897]: received mask_obs with header seq: 135 +stamp: + secs: 1735696096 + nsecs: 83663463 +frame_id: '' and grasped_object_mask header seq: 101968 +stamp: + secs: 1735696095 + nsecs: 877921820 +frame_id: "camera_color_optical_frame" +[INFO] [1735696096.272405]: received mask_obs with header seq: 136 +stamp: + secs: 1735696096 + nsecs: 273454666 +frame_id: '' and grasped_object_mask header seq: 101974 +stamp: + secs: 1735696096 + nsecs: 77757359 +frame_id: "camera_color_optical_frame" +[INFO] [1735696096.476512]: received mask_obs with header seq: 137 +stamp: + secs: 1735696096 + nsecs: 477650642 +frame_id: '' and grasped_object_mask header seq: 101980 +stamp: + secs: 1735696096 + nsecs: 277594090 +frame_id: "camera_color_optical_frame" +[INFO] [1735696096.680604]: received mask_obs with header seq: 138 +stamp: + secs: 1735696096 + nsecs: 681682348 +frame_id: '' and grasped_object_mask header seq: 101986 +stamp: + secs: 1735696096 + nsecs: 477433681 +frame_id: "camera_color_optical_frame" +[INFO] [1735696096.886847]: received mask_obs with header seq: 139 +stamp: + secs: 1735696096 + nsecs: 887973785 +frame_id: '' and grasped_object_mask header seq: 101992 +stamp: + secs: 1735696096 + nsecs: 677273273 +frame_id: "camera_color_optical_frame" +[INFO] [1735696097.092904]: received mask_obs with header seq: 140 +stamp: + secs: 1735696097 + nsecs: 92042684 +frame_id: '' and grasped_object_mask header seq: 101998 +stamp: + secs: 1735696096 + nsecs: 877113819 +frame_id: "camera_color_optical_frame" +[INFO] [1735696097.310795]: received mask_obs with header seq: 141 +stamp: + secs: 1735696097 + nsecs: 311478614 +frame_id: '' and grasped_object_mask header seq: 102004 +stamp: + secs: 1735696097 + nsecs: 76953650 +frame_id: "camera_color_optical_frame" +[INFO] [1735696097.512275]: received mask_obs with header seq: 142 +stamp: + secs: 1735696097 + nsecs: 513432264 +frame_id: '' and grasped_object_mask header seq: 102010 +stamp: + secs: 1735696097 + nsecs: 276790142 +frame_id: "camera_color_optical_frame" +[INFO] [1735696097.707104]: received mask_obs with header seq: 143 +stamp: + secs: 1735696097 + nsecs: 708178281 +frame_id: '' and grasped_object_mask header seq: 102016 +stamp: + secs: 1735696097 + nsecs: 476620197 +frame_id: "camera_color_optical_frame" +[INFO] [1735696097.904334]: received mask_obs with header seq: 144 +stamp: + secs: 1735696097 + nsecs: 905420780 +frame_id: '' and grasped_object_mask header seq: 102022 +stamp: + secs: 1735696097 + nsecs: 676449776 +frame_id: "camera_color_optical_frame" +[INFO] [1735696098.104532]: received mask_obs with header seq: 145 +stamp: + secs: 1735696098 + nsecs: 105592489 +frame_id: '' and grasped_object_mask header seq: 102028 +stamp: + secs: 1735696097 + nsecs: 876278877 +frame_id: "camera_color_optical_frame" +[INFO] [1735696098.306418]: received mask_obs with header seq: 146 +stamp: + secs: 1735696098 + nsecs: 307482957 +frame_id: '' and grasped_object_mask header seq: 102034 +stamp: + secs: 1735696098 + nsecs: 76108456 +frame_id: "camera_color_optical_frame" +[INFO] [1735696098.500184]: received mask_obs with header seq: 147 +stamp: + secs: 1735696098 + nsecs: 501182556 +frame_id: '' and grasped_object_mask header seq: 102040 +stamp: + secs: 1735696098 + nsecs: 275938034 +frame_id: "camera_color_optical_frame" +[INFO] [1735696098.698090]: received mask_obs with header seq: 148 +stamp: + secs: 1735696098 + nsecs: 699202775 +frame_id: '' and grasped_object_mask header seq: 102046 +stamp: + secs: 1735696098 + nsecs: 475770235 +frame_id: "camera_color_optical_frame" +[INFO] [1735696098.898207]: received mask_obs with header seq: 149 +stamp: + secs: 1735696098 + nsecs: 898722410 +frame_id: '' and grasped_object_mask header seq: 102052 +stamp: + secs: 1735696098 + nsecs: 675601959 +frame_id: "camera_color_optical_frame" +[INFO] [1735696099.097241]: received mask_obs with header seq: 150 +stamp: + secs: 1735696099 + nsecs: 98396301 +frame_id: '' and grasped_object_mask header seq: 102058 +stamp: + secs: 1735696098 + nsecs: 875433922 +frame_id: "camera_color_optical_frame" +[INFO] [1735696099.299214]: received mask_obs with header seq: 151 +stamp: + secs: 1735696099 + nsecs: 300285339 +frame_id: '' and grasped_object_mask header seq: 102064 +stamp: + secs: 1735696099 + nsecs: 75265408 +frame_id: "camera_color_optical_frame" +[INFO] [1735696099.496777]: received mask_obs with header seq: 152 +stamp: + secs: 1735696099 + nsecs: 497636079 +frame_id: '' and grasped_object_mask header seq: 102070 +stamp: + secs: 1735696099 + nsecs: 275097132 +frame_id: "camera_color_optical_frame" +[INFO] [1735696099.695699]: received mask_obs with header seq: 153 +stamp: + secs: 1735696099 + nsecs: 696844577 +frame_id: '' and grasped_object_mask header seq: 102076 +stamp: + secs: 1735696099 + nsecs: 474926710 +frame_id: "camera_color_optical_frame" +[INFO] [1735696099.895401]: received mask_obs with header seq: 154 +stamp: + secs: 1735696099 + nsecs: 896555185 +frame_id: '' and grasped_object_mask header seq: 102082 +stamp: + secs: 1735696099 + nsecs: 674755573 +frame_id: "camera_color_optical_frame" +[INFO] [1735696100.096196]: received mask_obs with header seq: 155 +stamp: + secs: 1735696100 + nsecs: 97412347 +frame_id: '' and grasped_object_mask header seq: 102088 +stamp: + secs: 1735696099 + nsecs: 874583960 +frame_id: "camera_color_optical_frame" +[INFO] [1735696100.301136]: received mask_obs with header seq: 156 +stamp: + secs: 1735696100 + nsecs: 301855325 +frame_id: '' and grasped_object_mask header seq: 102094 +stamp: + secs: 1735696100 + nsecs: 74411869 +frame_id: "camera_color_optical_frame" +[INFO] [1735696100.504331]: received mask_obs with header seq: 157 +stamp: + secs: 1735696100 + nsecs: 505494356 +frame_id: '' and grasped_object_mask header seq: 102100 +stamp: + secs: 1735696100 + nsecs: 274242401 +frame_id: "camera_color_optical_frame" +[INFO] [1735696100.701202]: received mask_obs with header seq: 158 +stamp: + secs: 1735696100 + nsecs: 702355146 +frame_id: '' and grasped_object_mask header seq: 102106 +stamp: + secs: 1735696100 + nsecs: 474080086 +frame_id: "camera_color_optical_frame" +[INFO] [1735696100.898086]: received mask_obs with header seq: 159 +stamp: + secs: 1735696100 + nsecs: 899000167 +frame_id: '' and grasped_object_mask header seq: 102112 +stamp: + secs: 1735696100 + nsecs: 673916817 +frame_id: "camera_color_optical_frame" +[INFO] [1735696101.100064]: received mask_obs with header seq: 160 +stamp: + secs: 1735696101 + nsecs: 100955963 +frame_id: '' and grasped_object_mask header seq: 102118 +stamp: + secs: 1735696100 + nsecs: 873753071 +frame_id: "camera_color_optical_frame" +[INFO] [1735696101.297143]: received mask_obs with header seq: 161 +stamp: + secs: 1735696101 + nsecs: 298304796 +frame_id: '' and grasped_object_mask header seq: 102124 +stamp: + secs: 1735696101 + nsecs: 73589563 +frame_id: "camera_color_optical_frame" +[INFO] [1735696101.495262]: received mask_obs with header seq: 162 +stamp: + secs: 1735696101 + nsecs: 496465206 +frame_id: '' and grasped_object_mask header seq: 102130 +stamp: + secs: 1735696101 + nsecs: 273426533 +frame_id: "camera_color_optical_frame" +[INFO] [1735696101.696579]: received mask_obs with header seq: 163 +stamp: + secs: 1735696101 + nsecs: 697786331 +frame_id: '' and grasped_object_mask header seq: 102136 +stamp: + secs: 1735696101 + nsecs: 473264694 +frame_id: "camera_color_optical_frame" +[INFO] [1735696101.898855]: received mask_obs with header seq: 164 +stamp: + secs: 1735696101 + nsecs: 900000572 +frame_id: '' and grasped_object_mask header seq: 102142 +stamp: + secs: 1735696101 + nsecs: 673103094 +frame_id: "camera_color_optical_frame" +[INFO] [1735696102.095778]: received mask_obs with header seq: 165 +stamp: + secs: 1735696102 + nsecs: 96912860 +frame_id: '' and grasped_object_mask header seq: 102148 +stamp: + secs: 1735696101 + nsecs: 872942209 +frame_id: "camera_color_optical_frame" +[INFO] [1735696102.294522]: received mask_obs with header seq: 166 +stamp: + secs: 1735696102 + nsecs: 295450925 +frame_id: '' and grasped_object_mask header seq: 102154 +stamp: + secs: 1735696102 + nsecs: 72780609 +frame_id: "camera_color_optical_frame" +[INFO] [1735696102.495227]: received mask_obs with header seq: 167 +stamp: + secs: 1735696102 + nsecs: 496391534 +frame_id: '' and grasped_object_mask header seq: 102160 +stamp: + secs: 1735696102 + nsecs: 272617579 +frame_id: "camera_color_optical_frame" +[INFO] [1735696102.694290]: received mask_obs with header seq: 168 +stamp: + secs: 1735696102 + nsecs: 695005655 +frame_id: '' and grasped_object_mask header seq: 102166 +stamp: + secs: 1735696102 + nsecs: 472453356 +frame_id: "camera_color_optical_frame" +[INFO] [1735696102.893700]: received mask_obs with header seq: 169 +stamp: + secs: 1735696102 + nsecs: 894484519 +frame_id: '' and grasped_object_mask header seq: 102172 +stamp: + secs: 1735696102 + nsecs: 672288656 +frame_id: "camera_color_optical_frame" +[INFO] [1735696103.096651]: received mask_obs with header seq: 170 +stamp: + secs: 1735696103 + nsecs: 97773551 +frame_id: '' and grasped_object_mask header seq: 102178 +stamp: + secs: 1735696102 + nsecs: 872123718 +frame_id: "camera_color_optical_frame" +[INFO] [1735696103.292730]: received mask_obs with header seq: 171 +stamp: + secs: 1735696103 + nsecs: 293938159 +frame_id: '' and grasped_object_mask header seq: 102184 +stamp: + secs: 1735696103 + nsecs: 71959734 +frame_id: "camera_color_optical_frame" +[INFO] [1735696103.494319]: received mask_obs with header seq: 172 +stamp: + secs: 1735696103 + nsecs: 494426727 +frame_id: '' and grasped_object_mask header seq: 102190 +stamp: + secs: 1735696103 + nsecs: 271796703 +frame_id: "camera_color_optical_frame" +[INFO] [1735696103.693755]: received mask_obs with header seq: 173 +stamp: + secs: 1735696103 + nsecs: 694412708 +frame_id: '' and grasped_object_mask header seq: 102196 +stamp: + secs: 1735696103 + nsecs: 471637249 +frame_id: "camera_color_optical_frame" +[INFO] [1735696103.894430]: received mask_obs with header seq: 174 +stamp: + secs: 1735696103 + nsecs: 895204544 +frame_id: '' and grasped_object_mask header seq: 102202 +stamp: + secs: 1735696103 + nsecs: 671478033 +frame_id: "camera_color_optical_frame" +[INFO] [1735696104.096848]: received mask_obs with header seq: 175 +stamp: + secs: 1735696104 + nsecs: 97990751 +frame_id: '' and grasped_object_mask header seq: 102208 +stamp: + secs: 1735696103 + nsecs: 871319056 +frame_id: "camera_color_optical_frame" +[INFO] [1735696104.294558]: received mask_obs with header seq: 176 +stamp: + secs: 1735696104 + nsecs: 295631647 +frame_id: '' and grasped_object_mask header seq: 102214 +stamp: + secs: 1735696104 + nsecs: 71160078 +frame_id: "camera_color_optical_frame" +[INFO] [1735696104.497236]: received mask_obs with header seq: 177 +stamp: + secs: 1735696104 + nsecs: 498210668 +frame_id: '' and grasped_object_mask header seq: 102220 +stamp: + secs: 1735696104 + nsecs: 271001816 +frame_id: "camera_color_optical_frame" +[INFO] [1735696104.695307]: received mask_obs with header seq: 178 +stamp: + secs: 1735696104 + nsecs: 695799112 +frame_id: '' and grasped_object_mask header seq: 102226 +stamp: + secs: 1735696104 + nsecs: 470841408 +frame_id: "camera_color_optical_frame" +[INFO] [1735696104.894957]: received mask_obs with header seq: 179 +stamp: + secs: 1735696104 + nsecs: 896030902 +frame_id: '' and grasped_object_mask header seq: 102232 +stamp: + secs: 1735696104 + nsecs: 670681238 +frame_id: "camera_color_optical_frame" +[INFO] [1735696105.096028]: received mask_obs with header seq: 180 +stamp: + secs: 1735696105 + nsecs: 97202539 +frame_id: '' and grasped_object_mask header seq: 102238 +stamp: + secs: 1735696104 + nsecs: 870520830 +frame_id: "camera_color_optical_frame" +[INFO] [1735696105.294927]: received mask_obs with header seq: 181 +stamp: + secs: 1735696105 + nsecs: 295247316 +frame_id: '' and grasped_object_mask header seq: 102244 +stamp: + secs: 1735696105 + nsecs: 70360661 +frame_id: "camera_color_optical_frame" +[INFO] [1735696105.495013]: received mask_obs with header seq: 182 +stamp: + secs: 1735696105 + nsecs: 496046304 +frame_id: '' and grasped_object_mask header seq: 102250 +stamp: + secs: 1735696105 + nsecs: 270198584 +frame_id: "camera_color_optical_frame" +[INFO] [1735696105.697108]: received mask_obs with header seq: 183 +stamp: + secs: 1735696105 + nsecs: 698271751 +frame_id: '' and grasped_object_mask header seq: 102256 +stamp: + secs: 1735696105 + nsecs: 470032930 +frame_id: "camera_color_optical_frame" +[INFO] [1735696105.899877]: received mask_obs with header seq: 184 +stamp: + secs: 1735696105 + nsecs: 901072740 +frame_id: '' and grasped_object_mask header seq: 102262 +stamp: + secs: 1735696105 + nsecs: 669866562 +frame_id: "camera_color_optical_frame" +[INFO] [1735696106.097636]: received mask_obs with header seq: 185 +stamp: + secs: 1735696106 + nsecs: 98380327 +frame_id: '' and grasped_object_mask header seq: 102268 +stamp: + secs: 1735696105 + nsecs: 869699955 +frame_id: "camera_color_optical_frame" +[INFO] [1735696106.295512]: received mask_obs with header seq: 186 +stamp: + secs: 1735696106 + nsecs: 296570062 +frame_id: '' and grasped_object_mask header seq: 102274 +stamp: + secs: 1735696106 + nsecs: 69533110 +frame_id: "camera_color_optical_frame" +[INFO] [1735696106.495407]: received mask_obs with header seq: 187 +stamp: + secs: 1735696106 + nsecs: 496568918 +frame_id: '' and grasped_object_mask header seq: 102280 +stamp: + secs: 1735696106 + nsecs: 269366741 +frame_id: "camera_color_optical_frame" +[INFO] [1735696106.695382]: received mask_obs with header seq: 188 +stamp: + secs: 1735696106 + nsecs: 696225166 +frame_id: '' and grasped_object_mask header seq: 102286 +stamp: + secs: 1735696106 + nsecs: 469201088 +frame_id: "camera_color_optical_frame" +[INFO] [1735696106.894686]: received mask_obs with header seq: 189 +stamp: + secs: 1735696106 + nsecs: 895831823 +frame_id: '' and grasped_object_mask header seq: 102292 +stamp: + secs: 1735696106 + nsecs: 669036627 +frame_id: "camera_color_optical_frame" +[INFO] [1735696107.096777]: received mask_obs with header seq: 190 +stamp: + secs: 1735696107 + nsecs: 97794294 +frame_id: '' and grasped_object_mask header seq: 102298 +stamp: + secs: 1735696106 + nsecs: 868871212 +frame_id: "camera_color_optical_frame" +[INFO] [1735696107.296049]: received mask_obs with header seq: 191 +stamp: + secs: 1735696107 + nsecs: 296593904 +frame_id: '' and grasped_object_mask header seq: 102304 +stamp: + secs: 1735696107 + nsecs: 68705082 +frame_id: "camera_color_optical_frame" +[INFO] [1735696107.491554]: received mask_obs with header seq: 192 +stamp: + secs: 1735696107 + nsecs: 492753267 +frame_id: '' and grasped_object_mask header seq: 102310 +stamp: + secs: 1735696107 + nsecs: 268541098 +frame_id: "camera_color_optical_frame" +[INFO] [1735696107.694493]: received mask_obs with header seq: 193 +stamp: + secs: 1735696107 + nsecs: 694900989 +frame_id: '' and grasped_object_mask header seq: 102316 +stamp: + secs: 1735696107 + nsecs: 468380690 +frame_id: "camera_color_optical_frame" +[INFO] [1735696107.891817]: received mask_obs with header seq: 194 +stamp: + secs: 1735696107 + nsecs: 893010616 +frame_id: '' and grasped_object_mask header seq: 102322 +stamp: + secs: 1735696107 + nsecs: 668220282 +frame_id: "camera_color_optical_frame" +[INFO] [1735696108.095024]: received mask_obs with header seq: 195 +stamp: + secs: 1735696108 + nsecs: 96092939 +frame_id: '' and grasped_object_mask header seq: 102328 +stamp: + secs: 1735696107 + nsecs: 868061066 +frame_id: "camera_color_optical_frame" +[INFO] [1735696108.289750]: received mask_obs with header seq: 196 +stamp: + secs: 1735696108 + nsecs: 290969133 +frame_id: '' and grasped_object_mask header seq: 102334 +stamp: + secs: 1735696108 + nsecs: 67901134 +frame_id: "camera_color_optical_frame" +[INFO] [1735696108.491098]: received mask_obs with header seq: 197 +stamp: + secs: 1735696108 + nsecs: 492280006 +frame_id: '' and grasped_object_mask header seq: 102340 +stamp: + secs: 1735696108 + nsecs: 267740250 +frame_id: "camera_color_optical_frame" +[INFO] [1735696108.692252]: received mask_obs with header seq: 198 +stamp: + secs: 1735696108 + nsecs: 693478107 +frame_id: '' and grasped_object_mask header seq: 102346 +stamp: + secs: 1735696108 + nsecs: 467577696 +frame_id: "camera_color_optical_frame" +[INFO] [1735696108.891526]: received mask_obs with header seq: 199 +stamp: + secs: 1735696108 + nsecs: 892712593 +frame_id: '' and grasped_object_mask header seq: 102352 +stamp: + secs: 1735696108 + nsecs: 667415380 +frame_id: "camera_color_optical_frame" +[INFO] [1735696109.092032]: received mask_obs with header seq: 200 +stamp: + secs: 1735696109 + nsecs: 92975854 +frame_id: '' and grasped_object_mask header seq: 102358 +stamp: + secs: 1735696108 + nsecs: 867252827 +frame_id: "camera_color_optical_frame" +[INFO] [1735696109.291628]: received mask_obs with header seq: 201 +stamp: + secs: 1735696109 + nsecs: 292804956 +frame_id: '' and grasped_object_mask header seq: 102364 +stamp: + secs: 1735696109 + nsecs: 67090273 +frame_id: "camera_color_optical_frame" +[INFO] [1735696109.490893]: received mask_obs with header seq: 202 +stamp: + secs: 1735696109 + nsecs: 491101980 +frame_id: '' and grasped_object_mask header seq: 102370 +stamp: + secs: 1735696109 + nsecs: 266927958 +frame_id: "camera_color_optical_frame" +[INFO] [1735696109.690958]: received mask_obs with header seq: 203 +stamp: + secs: 1735696109 + nsecs: 691561698 +frame_id: '' and grasped_object_mask header seq: 102376 +stamp: + secs: 1735696109 + nsecs: 466763020 +frame_id: "camera_color_optical_frame" +[INFO] [1735696109.891073]: received mask_obs with header seq: 204 +stamp: + secs: 1735696109 + nsecs: 892277240 +frame_id: '' and grasped_object_mask header seq: 102382 +stamp: + secs: 1735696109 + nsecs: 666597843 +frame_id: "camera_color_optical_frame" +[INFO] [1735696110.097152]: received mask_obs with header seq: 205 +stamp: + secs: 1735696110 + nsecs: 98334312 +frame_id: '' and grasped_object_mask header seq: 102388 +stamp: + secs: 1735696109 + nsecs: 866432905 +frame_id: "camera_color_optical_frame" +[INFO] [1735696110.292560]: received mask_obs with header seq: 206 +stamp: + secs: 1735696110 + nsecs: 293782711 +frame_id: '' and grasped_object_mask header seq: 102394 +stamp: + secs: 1735696110 + nsecs: 66267490 +frame_id: "camera_color_optical_frame" +[INFO] [1735696110.491582]: received mask_obs with header seq: 207 +stamp: + secs: 1735696110 + nsecs: 492586612 +frame_id: '' and grasped_object_mask header seq: 102400 +stamp: + secs: 1735696110 + nsecs: 266103506 +frame_id: "camera_color_optical_frame" +[INFO] [1735696110.688784]: received mask_obs with header seq: 208 +stamp: + secs: 1735696110 + nsecs: 689982414 +frame_id: '' and grasped_object_mask header seq: 102406 +stamp: + secs: 1735696110 + nsecs: 465942383 +frame_id: "camera_color_optical_frame" +[INFO] [1735696110.889658]: received mask_obs with header seq: 209 +stamp: + secs: 1735696110 + nsecs: 890759706 +frame_id: '' and grasped_object_mask header seq: 102412 +stamp: + secs: 1735696110 + nsecs: 665780544 +frame_id: "camera_color_optical_frame" +[INFO] [1735696111.096490]: received mask_obs with header seq: 210 +stamp: + secs: 1735696111 + nsecs: 97501993 +frame_id: '' and grasped_object_mask header seq: 102418 +stamp: + secs: 1735696110 + nsecs: 865618706 +frame_id: "camera_color_optical_frame" +[INFO] [1735696111.292068]: received mask_obs with header seq: 211 +stamp: + secs: 1735696111 + nsecs: 293270111 +frame_id: '' and grasped_object_mask header seq: 102424 +stamp: + secs: 1735696111 + nsecs: 65457106 +frame_id: "camera_color_optical_frame" +[INFO] [1735696111.490002]: received mask_obs with header seq: 212 +stamp: + secs: 1735696111 + nsecs: 490784883 +frame_id: '' and grasped_object_mask header seq: 102430 +stamp: + secs: 1735696111 + nsecs: 265297174 +frame_id: "camera_color_optical_frame" +[INFO] [1735696111.691241]: received mask_obs with header seq: 213 +stamp: + secs: 1735696111 + nsecs: 692470312 +frame_id: '' and grasped_object_mask header seq: 102436 +stamp: + secs: 1735696111 + nsecs: 465140581 +frame_id: "camera_color_optical_frame" +[INFO] [1735696111.889838]: received mask_obs with header seq: 214 +stamp: + secs: 1735696111 + nsecs: 891073465 +frame_id: '' and grasped_object_mask header seq: 102442 +stamp: + secs: 1735696111 + nsecs: 664985418 +frame_id: "camera_color_optical_frame" +[INFO] [1735696112.090783]: received mask_obs with header seq: 215 +stamp: + secs: 1735696112 + nsecs: 92007637 +frame_id: '' and grasped_object_mask header seq: 102448 +stamp: + secs: 1735696111 + nsecs: 864829540 +frame_id: "camera_color_optical_frame" +[INFO] [1735696112.287866]: received mask_obs with header seq: 216 +stamp: + secs: 1735696112 + nsecs: 289002180 +frame_id: '' and grasped_object_mask header seq: 102454 +stamp: + secs: 1735696112 + nsecs: 64673901 +frame_id: "camera_color_optical_frame" +[INFO] [1735696112.488043]: received mask_obs with header seq: 217 +stamp: + secs: 1735696112 + nsecs: 489294290 +frame_id: '' and grasped_object_mask header seq: 102460 +stamp: + secs: 1735696112 + nsecs: 264517784 +frame_id: "camera_color_optical_frame" +[INFO] [1735696112.688760]: received mask_obs with header seq: 218 +stamp: + secs: 1735696112 + nsecs: 689812183 +frame_id: '' and grasped_object_mask header seq: 102466 +stamp: + secs: 1735696112 + nsecs: 464360952 +frame_id: "camera_color_optical_frame" +[INFO] [1735696112.888420]: received mask_obs with header seq: 219 +stamp: + secs: 1735696112 + nsecs: 889697074 +frame_id: '' and grasped_object_mask header seq: 102472 +stamp: + secs: 1735696112 + nsecs: 664204121 +frame_id: "camera_color_optical_frame" +[INFO] [1735696113.089826]: received mask_obs with header seq: 220 +stamp: + secs: 1735696113 + nsecs: 91078758 +frame_id: '' and grasped_object_mask header seq: 102478 +stamp: + secs: 1735696112 + nsecs: 864048481 +frame_id: "camera_color_optical_frame" +[INFO] [1735696113.289458]: received mask_obs with header seq: 221 +stamp: + secs: 1735696113 + nsecs: 290412902 +frame_id: '' and grasped_object_mask header seq: 102484 +stamp: + secs: 1735696113 + nsecs: 63892365 +frame_id: "camera_color_optical_frame" +[INFO] [1735696113.487041]: received mask_obs with header seq: 222 +stamp: + secs: 1735696113 + nsecs: 488223075 +frame_id: '' and grasped_object_mask header seq: 102490 +stamp: + secs: 1735696113 + nsecs: 263736010 +frame_id: "camera_color_optical_frame" +[INFO] [1735696113.688608]: received mask_obs with header seq: 223 +stamp: + secs: 1735696113 + nsecs: 689877033 +frame_id: '' and grasped_object_mask header seq: 102496 +stamp: + secs: 1735696113 + nsecs: 463579655 +frame_id: "camera_color_optical_frame" +[INFO] [1735696113.889255]: received mask_obs with header seq: 224 +stamp: + secs: 1735696113 + nsecs: 890098333 +frame_id: '' and grasped_object_mask header seq: 102502 +stamp: + secs: 1735696113 + nsecs: 663423061 +frame_id: "camera_color_optical_frame" +[INFO] [1735696114.089107]: received mask_obs with header seq: 225 +stamp: + secs: 1735696114 + nsecs: 90083599 +frame_id: '' and grasped_object_mask header seq: 102508 +stamp: + secs: 1735696113 + nsecs: 863266945 +frame_id: "camera_color_optical_frame" +[INFO] [1735696114.288450]: received mask_obs with header seq: 226 +stamp: + secs: 1735696114 + nsecs: 289721012 +frame_id: '' and grasped_object_mask header seq: 102514 +stamp: + secs: 1735696114 + nsecs: 63111782 +frame_id: "camera_color_optical_frame" +[INFO] [1735696114.487647]: received mask_obs with header seq: 227 +stamp: + secs: 1735696114 + nsecs: 488707542 +frame_id: '' and grasped_object_mask header seq: 102520 +stamp: + secs: 1735696114 + nsecs: 262953520 +frame_id: "camera_color_optical_frame" +[INFO] [1735696114.689117]: received mask_obs with header seq: 228 +stamp: + secs: 1735696114 + nsecs: 690385818 +frame_id: '' and grasped_object_mask header seq: 102526 +stamp: + secs: 1735696114 + nsecs: 462790728 +frame_id: "camera_color_optical_frame" +[INFO] [1735696114.891979]: received mask_obs with header seq: 229 +stamp: + secs: 1735696114 + nsecs: 893254041 +frame_id: '' and grasped_object_mask header seq: 102532 +stamp: + secs: 1735696114 + nsecs: 662627697 +frame_id: "camera_color_optical_frame" +[INFO] [1735696115.087667]: received mask_obs with header seq: 230 +stamp: + secs: 1735696115 + nsecs: 88954687 +frame_id: '' and grasped_object_mask header seq: 102538 +stamp: + secs: 1735696114 + nsecs: 862464428 +frame_id: "camera_color_optical_frame" +[INFO] [1735696115.287850]: received mask_obs with header seq: 231 +stamp: + secs: 1735696115 + nsecs: 288920640 +frame_id: '' and grasped_object_mask header seq: 102544 +stamp: + secs: 1735696115 + nsecs: 62300920 +frame_id: "camera_color_optical_frame" +[INFO] [1735696115.487271]: received mask_obs with header seq: 232 +stamp: + secs: 1735696115 + nsecs: 488498926 +frame_id: '' and grasped_object_mask header seq: 102550 +stamp: + secs: 1735696115 + nsecs: 262138128 +frame_id: "camera_color_optical_frame" +[INFO] [1735696115.687561]: received mask_obs with header seq: 233 +stamp: + secs: 1735696115 + nsecs: 688394784 +frame_id: '' and grasped_object_mask header seq: 102556 +stamp: + secs: 1735696115 + nsecs: 461973906 +frame_id: "camera_color_optical_frame" +[INFO] [1735696115.887247]: received mask_obs with header seq: 234 +stamp: + secs: 1735696115 + nsecs: 888521432 +frame_id: '' and grasped_object_mask header seq: 102562 +stamp: + secs: 1735696115 + nsecs: 661809444 +frame_id: "camera_color_optical_frame" +[INFO] [1735696116.088052]: received mask_obs with header seq: 235 +stamp: + secs: 1735696116 + nsecs: 88818788 +frame_id: '' and grasped_object_mask header seq: 102568 +stamp: + secs: 1735696115 + nsecs: 861644745 +frame_id: "camera_color_optical_frame" +[INFO] [1735696116.286168]: received mask_obs with header seq: 236 +stamp: + secs: 1735696116 + nsecs: 287331581 +frame_id: '' and grasped_object_mask header seq: 102574 +stamp: + secs: 1735696116 + nsecs: 61480284 +frame_id: "camera_color_optical_frame" +[INFO] [1735696116.487680]: received mask_obs with header seq: 237 +stamp: + secs: 1735696116 + nsecs: 488863229 +frame_id: '' and grasped_object_mask header seq: 102580 +stamp: + secs: 1735696116 + nsecs: 261316061 +frame_id: "camera_color_optical_frame" +[INFO] [1735696116.687154]: received mask_obs with header seq: 238 +stamp: + secs: 1735696116 + nsecs: 688383340 +frame_id: '' and grasped_object_mask header seq: 102586 +stamp: + secs: 1735696116 + nsecs: 461154699 +frame_id: "camera_color_optical_frame" +[INFO] [1735696116.886847]: received mask_obs with header seq: 239 +stamp: + secs: 1735696116 + nsecs: 888000488 +frame_id: '' and grasped_object_mask header seq: 102592 +stamp: + secs: 1735696116 + nsecs: 660994291 +frame_id: "camera_color_optical_frame" +[INFO] [1735696117.089593]: received mask_obs with header seq: 240 +stamp: + secs: 1735696117 + nsecs: 90291500 +frame_id: '' and grasped_object_mask header seq: 102598 +stamp: + secs: 1735696116 + nsecs: 860832691 +frame_id: "camera_color_optical_frame" +[INFO] [1735696117.288232]: received mask_obs with header seq: 241 +stamp: + secs: 1735696117 + nsecs: 289225101 +frame_id: '' and grasped_object_mask header seq: 102604 +stamp: + secs: 1735696117 + nsecs: 60671329 +frame_id: "camera_color_optical_frame" +[INFO] [1735696117.488136]: received mask_obs with header seq: 242 +stamp: + secs: 1735696117 + nsecs: 489028453 +frame_id: '' and grasped_object_mask header seq: 102610 +stamp: + secs: 1735696117 + nsecs: 260508299 +frame_id: "camera_color_optical_frame" +[INFO] [1735696117.689138]: received mask_obs with header seq: 243 +stamp: + secs: 1735696117 + nsecs: 690302133 +frame_id: '' and grasped_object_mask header seq: 102616 +stamp: + secs: 1735696117 + nsecs: 460340977 +frame_id: "camera_color_optical_frame" +[INFO] [1735696117.887641]: received mask_obs with header seq: 244 +stamp: + secs: 1735696117 + nsecs: 888760089 +frame_id: '' and grasped_object_mask header seq: 102622 +stamp: + secs: 1735696117 + nsecs: 660173893 +frame_id: "camera_color_optical_frame" +[INFO] [1735696118.089285]: received mask_obs with header seq: 245 +stamp: + secs: 1735696118 + nsecs: 90571165 +frame_id: '' and grasped_object_mask header seq: 102628 +stamp: + secs: 1735696117 + nsecs: 860007524 +frame_id: "camera_color_optical_frame" +[INFO] [1735696118.288811]: received mask_obs with header seq: 246 +stamp: + secs: 1735696118 + nsecs: 289412021 +frame_id: '' and grasped_object_mask header seq: 102634 +stamp: + secs: 1735696118 + nsecs: 59839725 +frame_id: "camera_color_optical_frame" +[INFO] [1735696118.488217]: received mask_obs with header seq: 247 +stamp: + secs: 1735696118 + nsecs: 489476203 +frame_id: '' and grasped_object_mask header seq: 102640 +stamp: + secs: 1735696118 + nsecs: 259672642 +frame_id: "camera_color_optical_frame" +[INFO] [1735696118.689296]: received mask_obs with header seq: 248 +stamp: + secs: 1735696118 + nsecs: 690434217 +frame_id: '' and grasped_object_mask header seq: 102646 +stamp: + secs: 1735696118 + nsecs: 459506989 +frame_id: "camera_color_optical_frame" +[INFO] [1735696118.888217]: received mask_obs with header seq: 249 +stamp: + secs: 1735696118 + nsecs: 888897895 +frame_id: '' and grasped_object_mask header seq: 102652 +stamp: + secs: 1735696118 + nsecs: 659341335 +frame_id: "camera_color_optical_frame" +[INFO] [1735696119.089510]: received mask_obs with header seq: 250 +stamp: + secs: 1735696119 + nsecs: 90286731 +frame_id: '' and grasped_object_mask header seq: 102658 +stamp: + secs: 1735696118 + nsecs: 859175444 +frame_id: "camera_color_optical_frame" +[INFO] [1735696119.288219]: received mask_obs with header seq: 251 +stamp: + secs: 1735696119 + nsecs: 288429021 +frame_id: '' and grasped_object_mask header seq: 102664 +stamp: + secs: 1735696119 + nsecs: 59010506 +frame_id: "camera_color_optical_frame" +[INFO] [1735696119.486929]: received mask_obs with header seq: 252 +stamp: + secs: 1735696119 + nsecs: 487949848 +frame_id: '' and grasped_object_mask header seq: 102670 +stamp: + secs: 1735696119 + nsecs: 258846760 +frame_id: "camera_color_optical_frame" +[INFO] [1735696119.687683]: received mask_obs with header seq: 253 +stamp: + secs: 1735696119 + nsecs: 688513040 +frame_id: '' and grasped_object_mask header seq: 102676 +stamp: + secs: 1735696119 + nsecs: 458692789 +frame_id: "camera_color_optical_frame" +[INFO] [1735696119.887537]: received mask_obs with header seq: 254 +stamp: + secs: 1735696119 + nsecs: 888784408 +frame_id: '' and grasped_object_mask header seq: 102682 +stamp: + secs: 1735696119 + nsecs: 658539295 +frame_id: "camera_color_optical_frame" +[INFO] [1735696120.096590]: received mask_obs with header seq: 255 +stamp: + secs: 1735696120 + nsecs: 97618579 +frame_id: '' and grasped_object_mask header seq: 102688 +stamp: + secs: 1735696119 + nsecs: 858386278 +frame_id: "camera_color_optical_frame" +[INFO] [1735696120.294289]: received mask_obs with header seq: 256 +stamp: + secs: 1735696120 + nsecs: 295325756 +frame_id: '' and grasped_object_mask header seq: 102694 +stamp: + secs: 1735696120 + nsecs: 58233738 +frame_id: "camera_color_optical_frame" +[INFO] [1735696120.487104]: received mask_obs with header seq: 257 +stamp: + secs: 1735696120 + nsecs: 487352848 +frame_id: '' and grasped_object_mask header seq: 102700 +stamp: + secs: 1735696120 + nsecs: 258081436 +frame_id: "camera_color_optical_frame" +[INFO] [1735696120.689970]: received mask_obs with header seq: 258 +stamp: + secs: 1735696120 + nsecs: 691278934 +frame_id: '' and grasped_object_mask header seq: 102706 +stamp: + secs: 1735696120 + nsecs: 457924604 +frame_id: "camera_color_optical_frame" +[INFO] [1735696120.887924]: received mask_obs with header seq: 259 +stamp: + secs: 1735696120 + nsecs: 889234542 +frame_id: '' and grasped_object_mask header seq: 102712 +stamp: + secs: 1735696120 + nsecs: 657768011 +frame_id: "camera_color_optical_frame" +[INFO] [1735696121.089096]: received mask_obs with header seq: 260 +stamp: + secs: 1735696121 + nsecs: 90169191 +frame_id: '' and grasped_object_mask header seq: 102718 +stamp: + secs: 1735696120 + nsecs: 857611895 +frame_id: "camera_color_optical_frame" +[INFO] [1735696121.292721]: received mask_obs with header seq: 261 +stamp: + secs: 1735696121 + nsecs: 293537378 +frame_id: '' and grasped_object_mask header seq: 102724 +stamp: + secs: 1735696121 + nsecs: 57455778 +frame_id: "camera_color_optical_frame" +[INFO] [1735696121.491699]: received mask_obs with header seq: 262 +stamp: + secs: 1735696121 + nsecs: 492933988 +frame_id: '' and grasped_object_mask header seq: 102730 +stamp: + secs: 1735696121 + nsecs: 257297993 +frame_id: "camera_color_optical_frame" +[INFO] [1735696121.688823]: received mask_obs with header seq: 263 +stamp: + secs: 1735696121 + nsecs: 689991950 +frame_id: '' and grasped_object_mask header seq: 102736 +stamp: + secs: 1735696121 + nsecs: 457132339 +frame_id: "camera_color_optical_frame" +[INFO] [1735696121.891645]: received mask_obs with header seq: 264 +stamp: + secs: 1735696121 + nsecs: 892526865 +frame_id: '' and grasped_object_mask header seq: 102742 +stamp: + secs: 1735696121 + nsecs: 656965256 +frame_id: "camera_color_optical_frame" +[INFO] [1735696122.090300]: received mask_obs with header seq: 265 +stamp: + secs: 1735696122 + nsecs: 91197967 +frame_id: '' and grasped_object_mask header seq: 102748 +stamp: + secs: 1735696121 + nsecs: 856798172 +frame_id: "camera_color_optical_frame" +[INFO] [1735696122.139465]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735696122.294055]: received mask_obs with header seq: 266 +stamp: + secs: 1735696122 + nsecs: 292031288 +frame_id: '' and grasped_object_mask header seq: 102754 +stamp: + secs: 1735696122 + nsecs: 56630373 +frame_id: "camera_color_optical_frame" +[INFO] [1735696122.915845]: Stored episode 2. +[INFO] [1735696122.918866]: Saving video... +[INFO] [1735696123.118098]: Video saved! +Starting episode 2 +[INFO] [1735696123.155020]: resetting environment +[INFO] [1735696123.164482]: cleared current plan +[INFO] [1735696123.164681]: moving to home +[INFO] [1735696126.769790]: reached home +[INFO] [1735696126.773644]: reset action history +[INFO] [1735696129.546469]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735696142.993443]: resetting environment +[INFO] [1735696143.001899]: cleared current plan +[INFO] [1735696143.007695]: moving to home +[INFO] [1735696144.017447]: reached home +[INFO] [1735696144.021455]: reset action history +[INFO] [1735696146.791507]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/episode_rosbags/episode_2_2024-12-31-20-49-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1737022.log +started roslaunch server http://158.130.50.37:40219/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1737836] +process[print_text-8]: started with pid [1737837] +process[rosbag_record-9]: started with pid [1737861] +started bagging! +[INFO] [1735696147.501370]: received mask_obs with header seq: 267 +stamp: + secs: 1735696147 + nsecs: 502482652 +frame_id: '' and grasped_object_mask header seq: 103512 +stamp: + secs: 1735696147 + nsecs: 302870274 +frame_id: "camera_color_optical_frame" +[INFO] [1735696147.705572]: received mask_obs with header seq: 268 +stamp: + secs: 1735696147 + nsecs: 706726312 +frame_id: '' and grasped_object_mask header seq: 103518 +stamp: + secs: 1735696147 + nsecs: 502710581 +frame_id: "camera_color_optical_frame" +[INFO] [1735696147.910422]: received mask_obs with header seq: 269 +stamp: + secs: 1735696147 + nsecs: 911456823 +frame_id: '' and grasped_object_mask header seq: 103524 +stamp: + secs: 1735696147 + nsecs: 702549696 +frame_id: "camera_color_optical_frame" +[INFO] [1735696148.110795]: received mask_obs with header seq: 270 +stamp: + secs: 1735696148 + nsecs: 112138032 +frame_id: '' and grasped_object_mask header seq: 103530 +stamp: + secs: 1735696147 + nsecs: 902388811 +frame_id: "camera_color_optical_frame" +[INFO] [1735696148.312546]: received mask_obs with header seq: 271 +stamp: + secs: 1735696148 + nsecs: 313905477 +frame_id: '' and grasped_object_mask header seq: 103536 +stamp: + secs: 1735696148 + nsecs: 102227926 +frame_id: "camera_color_optical_frame" +[INFO] [1735696148.521661]: received mask_obs with header seq: 272 +stamp: + secs: 1735696148 + nsecs: 522314548 +frame_id: '' and grasped_object_mask header seq: 103542 +stamp: + secs: 1735696148 + nsecs: 302068710 +frame_id: "camera_color_optical_frame" +[INFO] [1735696148.730967]: received mask_obs with header seq: 273 +stamp: + secs: 1735696148 + nsecs: 732117414 +frame_id: '' and grasped_object_mask header seq: 103548 +stamp: + secs: 1735696148 + nsecs: 501912355 +frame_id: "camera_color_optical_frame" +[INFO] [1735696148.939193]: received mask_obs with header seq: 274 +stamp: + secs: 1735696148 + nsecs: 940380334 +frame_id: '' and grasped_object_mask header seq: 103554 +stamp: + secs: 1735696148 + nsecs: 701757431 +frame_id: "camera_color_optical_frame" +[INFO] [1735696149.142328]: received mask_obs with header seq: 275 +stamp: + secs: 1735696149 + nsecs: 143684387 +frame_id: '' and grasped_object_mask header seq: 103560 +stamp: + secs: 1735696148 + nsecs: 901601791 +frame_id: "camera_color_optical_frame" +[INFO] [1735696149.336912]: received mask_obs with header seq: 276 +stamp: + secs: 1735696149 + nsecs: 338201761 +frame_id: '' and grasped_object_mask header seq: 103566 +stamp: + secs: 1735696149 + nsecs: 101446390 +frame_id: "camera_color_optical_frame" +[INFO] [1735696149.534573]: received mask_obs with header seq: 277 +stamp: + secs: 1735696149 + nsecs: 535861015 +frame_id: '' and grasped_object_mask header seq: 103572 +stamp: + secs: 1735696149 + nsecs: 301290035 +frame_id: "camera_color_optical_frame" +[INFO] [1735696149.729163]: received mask_obs with header seq: 278 +stamp: + secs: 1735696149 + nsecs: 730392932 +frame_id: '' and grasped_object_mask header seq: 103578 +stamp: + secs: 1735696149 + nsecs: 501132250 +frame_id: "camera_color_optical_frame" +[INFO] [1735696149.926680]: received mask_obs with header seq: 279 +stamp: + secs: 1735696149 + nsecs: 927746057 +frame_id: '' and grasped_object_mask header seq: 103584 +stamp: + secs: 1735696149 + nsecs: 700974941 +frame_id: "camera_color_optical_frame" +[INFO] [1735696150.130157]: received mask_obs with header seq: 280 +stamp: + secs: 1735696150 + nsecs: 131529331 +frame_id: '' and grasped_object_mask header seq: 103590 +stamp: + secs: 1735696149 + nsecs: 900818586 +frame_id: "camera_color_optical_frame" +[INFO] [1735696150.327718]: received mask_obs with header seq: 281 +stamp: + secs: 1735696150 + nsecs: 328681230 +frame_id: '' and grasped_object_mask header seq: 103596 +stamp: + secs: 1735696150 + nsecs: 100661516 +frame_id: "camera_color_optical_frame" +[INFO] [1735696150.528847]: received mask_obs with header seq: 282 +stamp: + secs: 1735696150 + nsecs: 530231714 +frame_id: '' and grasped_object_mask header seq: 103602 +stamp: + secs: 1735696150 + nsecs: 300503731 +frame_id: "camera_color_optical_frame" +[INFO] [1735696150.726428]: received mask_obs with header seq: 283 +stamp: + secs: 1735696150 + nsecs: 727677583 +frame_id: '' and grasped_object_mask header seq: 103608 +stamp: + secs: 1735696150 + nsecs: 500343323 +frame_id: "camera_color_optical_frame" +[INFO] [1735696150.928362]: received mask_obs with header seq: 284 +stamp: + secs: 1735696150 + nsecs: 929736375 +frame_id: '' and grasped_object_mask header seq: 103614 +stamp: + secs: 1735696150 + nsecs: 700182915 +frame_id: "camera_color_optical_frame" +[INFO] [1735696151.136379]: received mask_obs with header seq: 285 +stamp: + secs: 1735696151 + nsecs: 137765169 +frame_id: '' and grasped_object_mask header seq: 103620 +stamp: + secs: 1735696150 + nsecs: 900022745 +frame_id: "camera_color_optical_frame" +[INFO] [1735696151.324288]: received mask_obs with header seq: 286 +stamp: + secs: 1735696151 + nsecs: 325637340 +frame_id: '' and grasped_object_mask header seq: 103626 +stamp: + secs: 1735696151 + nsecs: 99863768 +frame_id: "camera_color_optical_frame" +[INFO] [1735696151.528949]: received mask_obs with header seq: 287 +stamp: + secs: 1735696151 + nsecs: 530148983 +frame_id: '' and grasped_object_mask header seq: 103632 +stamp: + secs: 1735696151 + nsecs: 299703360 +frame_id: "camera_color_optical_frame" +[INFO] [1735696151.726079]: received mask_obs with header seq: 288 +stamp: + secs: 1735696151 + nsecs: 727472305 +frame_id: '' and grasped_object_mask header seq: 103638 +stamp: + secs: 1735696151 + nsecs: 499542236 +frame_id: "camera_color_optical_frame" +[INFO] [1735696151.925416]: received mask_obs with header seq: 289 +stamp: + secs: 1735696151 + nsecs: 926833868 +frame_id: '' and grasped_object_mask header seq: 103644 +stamp: + secs: 1735696151 + nsecs: 699381113 +frame_id: "camera_color_optical_frame" +[INFO] [1735696152.128039]: received mask_obs with header seq: 290 +stamp: + secs: 1735696152 + nsecs: 129198312 +frame_id: '' and grasped_object_mask header seq: 103650 +stamp: + secs: 1735696151 + nsecs: 899219751 +frame_id: "camera_color_optical_frame" +[INFO] [1735696152.324237]: received mask_obs with header seq: 291 +stamp: + secs: 1735696152 + nsecs: 325630187 +frame_id: '' and grasped_object_mask header seq: 103656 +stamp: + secs: 1735696152 + nsecs: 99058390 +frame_id: "camera_color_optical_frame" +[INFO] [1735696152.526238]: received mask_obs with header seq: 292 +stamp: + secs: 1735696152 + nsecs: 527427434 +frame_id: '' and grasped_object_mask header seq: 103662 +stamp: + secs: 1735696152 + nsecs: 298898697 +frame_id: "camera_color_optical_frame" +[INFO] [1735696152.727685]: received mask_obs with header seq: 293 +stamp: + secs: 1735696152 + nsecs: 729069471 +frame_id: '' and grasped_object_mask header seq: 103668 +stamp: + secs: 1735696152 + nsecs: 498739243 +frame_id: "camera_color_optical_frame" +[INFO] [1735696152.929028]: received mask_obs with header seq: 294 +stamp: + secs: 1735696152 + nsecs: 930388450 +frame_id: '' and grasped_object_mask header seq: 103674 +stamp: + secs: 1735696152 + nsecs: 698579788 +frame_id: "camera_color_optical_frame" +[INFO] [1735696153.127498]: received mask_obs with header seq: 295 +stamp: + secs: 1735696153 + nsecs: 128869533 +frame_id: '' and grasped_object_mask header seq: 103680 +stamp: + secs: 1735696152 + nsecs: 898420572 +frame_id: "camera_color_optical_frame" +[INFO] [1735696153.325612]: received mask_obs with header seq: 296 +stamp: + secs: 1735696153 + nsecs: 327039718 +frame_id: '' and grasped_object_mask header seq: 103686 +stamp: + secs: 1735696153 + nsecs: 98261356 +frame_id: "camera_color_optical_frame" +[INFO] [1735696153.525113]: received mask_obs with header seq: 297 +stamp: + secs: 1735696153 + nsecs: 526112794 +frame_id: '' and grasped_object_mask header seq: 103692 +stamp: + secs: 1735696153 + nsecs: 298100471 +frame_id: "camera_color_optical_frame" +[INFO] [1735696153.724490]: received mask_obs with header seq: 298 +stamp: + secs: 1735696153 + nsecs: 725791215 +frame_id: '' and grasped_object_mask header seq: 103698 +stamp: + secs: 1735696153 + nsecs: 497936964 +frame_id: "camera_color_optical_frame" +[INFO] [1735696153.925514]: received mask_obs with header seq: 299 +stamp: + secs: 1735696153 + nsecs: 926701068 +frame_id: '' and grasped_object_mask header seq: 103704 +stamp: + secs: 1735696153 + nsecs: 697772741 +frame_id: "camera_color_optical_frame" +[INFO] [1735696154.126957]: received mask_obs with header seq: 300 +stamp: + secs: 1735696154 + nsecs: 128029346 +frame_id: '' and grasped_object_mask header seq: 103710 +stamp: + secs: 1735696153 + nsecs: 897607803 +frame_id: "camera_color_optical_frame" +[INFO] [1735696154.325556]: received mask_obs with header seq: 301 +stamp: + secs: 1735696154 + nsecs: 326873302 +frame_id: '' and grasped_object_mask header seq: 103716 +stamp: + secs: 1735696154 + nsecs: 97443104 +frame_id: "camera_color_optical_frame" +[INFO] [1735696154.528104]: received mask_obs with header seq: 302 +stamp: + secs: 1735696154 + nsecs: 529169082 +frame_id: '' and grasped_object_mask header seq: 103722 +stamp: + secs: 1735696154 + nsecs: 297280312 +frame_id: "camera_color_optical_frame" +[INFO] [1735696154.725361]: received mask_obs with header seq: 303 +stamp: + secs: 1735696154 + nsecs: 726641893 +frame_id: '' and grasped_object_mask header seq: 103728 +stamp: + secs: 1735696154 + nsecs: 497121572 +frame_id: "camera_color_optical_frame" +[INFO] [1735696154.925905]: received mask_obs with header seq: 304 +stamp: + secs: 1735696154 + nsecs: 927218198 +frame_id: '' and grasped_object_mask header seq: 103734 +stamp: + secs: 1735696154 + nsecs: 696963072 +frame_id: "camera_color_optical_frame" +[INFO] [1735696155.127778]: received mask_obs with header seq: 305 +stamp: + secs: 1735696155 + nsecs: 129146575 +frame_id: '' and grasped_object_mask header seq: 103740 +stamp: + secs: 1735696154 + nsecs: 896805763 +frame_id: "camera_color_optical_frame" +[INFO] [1735696155.325270]: received mask_obs with header seq: 306 +stamp: + secs: 1735696155 + nsecs: 326528787 +frame_id: '' and grasped_object_mask header seq: 103746 +stamp: + secs: 1735696155 + nsecs: 96647263 +frame_id: "camera_color_optical_frame" +[INFO] [1735696155.524232]: received mask_obs with header seq: 307 +stamp: + secs: 1735696155 + nsecs: 525603771 +frame_id: '' and grasped_object_mask header seq: 103752 +stamp: + secs: 1735696155 + nsecs: 296487808 +frame_id: "camera_color_optical_frame" +[INFO] [1735696155.726234]: received mask_obs with header seq: 308 +stamp: + secs: 1735696155 + nsecs: 727679729 +frame_id: '' and grasped_object_mask header seq: 103758 +stamp: + secs: 1735696155 + nsecs: 496325493 +frame_id: "camera_color_optical_frame" +[INFO] [1735696155.924623]: received mask_obs with header seq: 309 +stamp: + secs: 1735696155 + nsecs: 926000356 +frame_id: '' and grasped_object_mask header seq: 103764 +stamp: + secs: 1735696155 + nsecs: 696163177 +frame_id: "camera_color_optical_frame" +[INFO] [1735696156.126663]: received mask_obs with header seq: 310 +stamp: + secs: 1735696156 + nsecs: 127908468 +frame_id: '' and grasped_object_mask header seq: 103770 +stamp: + secs: 1735696155 + nsecs: 896000385 +frame_id: "camera_color_optical_frame" +[INFO] [1735696156.326049]: received mask_obs with header seq: 311 +stamp: + secs: 1735696156 + nsecs: 327383756 +frame_id: '' and grasped_object_mask header seq: 103776 +stamp: + secs: 1735696156 + nsecs: 95839024 +frame_id: "camera_color_optical_frame" +[INFO] [1735696156.525413]: received mask_obs with header seq: 312 +stamp: + secs: 1735696156 + nsecs: 526754856 +frame_id: '' and grasped_object_mask header seq: 103782 +stamp: + secs: 1735696156 + nsecs: 295678139 +frame_id: "camera_color_optical_frame" +[INFO] [1735696156.725673]: received mask_obs with header seq: 313 +stamp: + secs: 1735696156 + nsecs: 726960420 +frame_id: '' and grasped_object_mask header seq: 103788 +stamp: + secs: 1735696156 + nsecs: 495520592 +frame_id: "camera_color_optical_frame" +[INFO] [1735696156.926189]: received mask_obs with header seq: 314 +stamp: + secs: 1735696156 + nsecs: 927568912 +frame_id: '' and grasped_object_mask header seq: 103794 +stamp: + secs: 1735696156 + nsecs: 695363283 +frame_id: "camera_color_optical_frame" +[INFO] [1735696157.126200]: received mask_obs with header seq: 315 +stamp: + secs: 1735696157 + nsecs: 127575635 +frame_id: '' and grasped_object_mask header seq: 103800 +stamp: + secs: 1735696156 + nsecs: 895205736 +frame_id: "camera_color_optical_frame" +[INFO] [1735696157.324485]: received mask_obs with header seq: 316 +stamp: + secs: 1735696157 + nsecs: 325833797 +frame_id: '' and grasped_object_mask header seq: 103806 +stamp: + secs: 1735696157 + nsecs: 95048666 +frame_id: "camera_color_optical_frame" +[INFO] [1735696157.528231]: received mask_obs with header seq: 317 +stamp: + secs: 1735696157 + nsecs: 528482198 +frame_id: '' and grasped_object_mask header seq: 103812 +stamp: + secs: 1735696157 + nsecs: 294893265 +frame_id: "camera_color_optical_frame" +[INFO] [1735696157.726327]: received mask_obs with header seq: 318 +stamp: + secs: 1735696157 + nsecs: 727621316 +frame_id: '' and grasped_object_mask header seq: 103818 +stamp: + secs: 1735696157 + nsecs: 494739056 +frame_id: "camera_color_optical_frame" +[INFO] [1735696157.925400]: received mask_obs with header seq: 319 +stamp: + secs: 1735696157 + nsecs: 926774024 +frame_id: '' and grasped_object_mask header seq: 103824 +stamp: + secs: 1735696157 + nsecs: 694585323 +frame_id: "camera_color_optical_frame" +[INFO] [1735696158.128055]: received mask_obs with header seq: 320 +stamp: + secs: 1735696158 + nsecs: 129309892 +frame_id: '' and grasped_object_mask header seq: 103830 +stamp: + secs: 1735696157 + nsecs: 894431353 +frame_id: "camera_color_optical_frame" +[INFO] [1735696158.325186]: received mask_obs with header seq: 321 +stamp: + secs: 1735696158 + nsecs: 326593637 +frame_id: '' and grasped_object_mask header seq: 103836 +stamp: + secs: 1735696158 + nsecs: 94278097 +frame_id: "camera_color_optical_frame" +[INFO] [1735696158.530082]: received mask_obs with header seq: 322 +stamp: + secs: 1735696158 + nsecs: 531405448 +frame_id: '' and grasped_object_mask header seq: 103842 +stamp: + secs: 1735696158 + nsecs: 294121981 +frame_id: "camera_color_optical_frame" +[INFO] [1735696158.725244]: received mask_obs with header seq: 323 +stamp: + secs: 1735696158 + nsecs: 726641178 +frame_id: '' and grasped_object_mask header seq: 103848 +stamp: + secs: 1735696158 + nsecs: 493960857 +frame_id: "camera_color_optical_frame" +[INFO] [1735696158.927697]: received mask_obs with header seq: 324 +stamp: + secs: 1735696158 + nsecs: 929122447 +frame_id: '' and grasped_object_mask header seq: 103854 +stamp: + secs: 1735696158 + nsecs: 693798780 +frame_id: "camera_color_optical_frame" +[INFO] [1735696159.127132]: received mask_obs with header seq: 325 +stamp: + secs: 1735696159 + nsecs: 128378629 +frame_id: '' and grasped_object_mask header seq: 103860 +stamp: + secs: 1735696158 + nsecs: 893636465 +frame_id: "camera_color_optical_frame" +[INFO] [1735696159.330785]: received mask_obs with header seq: 326 +stamp: + secs: 1735696159 + nsecs: 332236051 +frame_id: '' and grasped_object_mask header seq: 103866 +stamp: + secs: 1735696159 + nsecs: 93474388 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -139931514 for nearest: 1735696159153380320 - target: 1735696159293311834 at idx 147 +[INFO] [1735696159.535225]: received mask_obs with header seq: 327 +stamp: + secs: 1735696159 + nsecs: 536668777 +frame_id: '' and grasped_object_mask header seq: 103872 +stamp: + secs: 1735696159 + nsecs: 293311834 +frame_id: "camera_color_optical_frame" +[INFO] [1735696159.731811]: received mask_obs with header seq: 328 +stamp: + secs: 1735696159 + nsecs: 733187913 +frame_id: '' and grasped_object_mask header seq: 103878 +stamp: + secs: 1735696159 + nsecs: 493148565 +frame_id: "camera_color_optical_frame" +[INFO] [1735696159.929331]: received mask_obs with header seq: 329 +stamp: + secs: 1735696159 + nsecs: 930658817 +frame_id: '' and grasped_object_mask header seq: 103884 +stamp: + secs: 1735696159 + nsecs: 692986250 +frame_id: "camera_color_optical_frame" +[INFO] [1735696160.128121]: received mask_obs with header seq: 330 +stamp: + secs: 1735696160 + nsecs: 129516124 +frame_id: '' and grasped_object_mask header seq: 103890 +stamp: + secs: 1735696159 + nsecs: 892822981 +frame_id: "camera_color_optical_frame" +[INFO] [1735696160.325317]: received mask_obs with header seq: 331 +stamp: + secs: 1735696160 + nsecs: 326604127 +frame_id: '' and grasped_object_mask header seq: 103896 +stamp: + secs: 1735696160 + nsecs: 92659473 +frame_id: "camera_color_optical_frame" +[INFO] [1735696160.527895]: received mask_obs with header seq: 332 +stamp: + secs: 1735696160 + nsecs: 527792930 +frame_id: '' and grasped_object_mask header seq: 103902 +stamp: + secs: 1735696160 + nsecs: 292495966 +frame_id: "camera_color_optical_frame" +[INFO] [1735696160.725629]: received mask_obs with header seq: 333 +stamp: + secs: 1735696160 + nsecs: 726992368 +frame_id: '' and grasped_object_mask header seq: 103908 +stamp: + secs: 1735696160 + nsecs: 492332220 +frame_id: "camera_color_optical_frame" +[INFO] [1735696160.937979]: received mask_obs with header seq: 334 +stamp: + secs: 1735696160 + nsecs: 939348697 +frame_id: '' and grasped_object_mask header seq: 103914 +stamp: + secs: 1735696160 + nsecs: 692168474 +frame_id: "camera_color_optical_frame" +[INFO] [1735696161.152331]: received mask_obs with header seq: 335 +stamp: + secs: 1735696161 + nsecs: 153589725 +frame_id: '' and grasped_object_mask header seq: 103920 +stamp: + secs: 1735696160 + nsecs: 892004490 +frame_id: "camera_color_optical_frame" +[INFO] [1735696161.346668]: received mask_obs with header seq: 336 +stamp: + secs: 1735696161 + nsecs: 348072528 +frame_id: '' and grasped_object_mask header seq: 103927 +stamp: + secs: 1735696161 + nsecs: 125146627 +frame_id: "camera_color_optical_frame" +[INFO] [1735696161.546153]: received mask_obs with header seq: 337 +stamp: + secs: 1735696161 + nsecs: 547615766 +frame_id: '' and grasped_object_mask header seq: 103933 +stamp: + secs: 1735696161 + nsecs: 324984312 +frame_id: "camera_color_optical_frame" +[INFO] [1735696161.747310]: received mask_obs with header seq: 338 +stamp: + secs: 1735696161 + nsecs: 748500823 +frame_id: '' and grasped_object_mask header seq: 103939 +stamp: + secs: 1735696161 + nsecs: 524825096 +frame_id: "camera_color_optical_frame" +[INFO] [1735696161.934867]: received mask_obs with header seq: 339 +stamp: + secs: 1735696161 + nsecs: 936205625 +frame_id: '' and grasped_object_mask header seq: 103944 +stamp: + secs: 1735696161 + nsecs: 691357613 +frame_id: "camera_color_optical_frame" +[INFO] [1735696162.130449]: received mask_obs with header seq: 340 +stamp: + secs: 1735696162 + nsecs: 131916284 +frame_id: '' and grasped_object_mask header seq: 103950 +stamp: + secs: 1735696161 + nsecs: 891197205 +frame_id: "camera_color_optical_frame" +[INFO] [1735696162.325759]: received mask_obs with header seq: 341 +stamp: + secs: 1735696162 + nsecs: 327131748 +frame_id: '' and grasped_object_mask header seq: 103956 +stamp: + secs: 1735696162 + nsecs: 91036797 +frame_id: "camera_color_optical_frame" +[INFO] [1735696162.527500]: received mask_obs with header seq: 342 +stamp: + secs: 1735696162 + nsecs: 528790712 +frame_id: '' and grasped_object_mask header seq: 103962 +stamp: + secs: 1735696162 + nsecs: 290876865 +frame_id: "camera_color_optical_frame" +[INFO] [1735696162.726057]: received mask_obs with header seq: 343 +stamp: + secs: 1735696162 + nsecs: 727245569 +frame_id: '' and grasped_object_mask header seq: 103968 +stamp: + secs: 1735696162 + nsecs: 490715027 +frame_id: "camera_color_optical_frame" +[INFO] [1735696162.929783]: received mask_obs with header seq: 344 +stamp: + secs: 1735696162 + nsecs: 931102752 +frame_id: '' and grasped_object_mask header seq: 103974 +stamp: + secs: 1735696162 + nsecs: 690552950 +frame_id: "camera_color_optical_frame" +[INFO] [1735696163.142849]: received mask_obs with header seq: 345 +stamp: + secs: 1735696163 + nsecs: 144268274 +frame_id: '' and grasped_object_mask header seq: 103980 +stamp: + secs: 1735696162 + nsecs: 890391350 +frame_id: "camera_color_optical_frame" +[INFO] [1735696163.330604]: received mask_obs with header seq: 346 +stamp: + secs: 1735696163 + nsecs: 331968069 +frame_id: '' and grasped_object_mask header seq: 103986 +stamp: + secs: 1735696163 + nsecs: 90229511 +frame_id: "camera_color_optical_frame" +[INFO] [1735696163.529794]: received mask_obs with header seq: 347 +stamp: + secs: 1735696163 + nsecs: 531199932 +frame_id: '' and grasped_object_mask header seq: 103992 +stamp: + secs: 1735696163 + nsecs: 290068626 +frame_id: "camera_color_optical_frame" +[INFO] [1735696163.727291]: received mask_obs with header seq: 348 +stamp: + secs: 1735696163 + nsecs: 728711366 +frame_id: '' and grasped_object_mask header seq: 103998 +stamp: + secs: 1735696163 + nsecs: 489910841 +frame_id: "camera_color_optical_frame" +[INFO] [1735696163.946497]: received mask_obs with header seq: 349 +stamp: + secs: 1735696163 + nsecs: 947915792 +frame_id: '' and grasped_object_mask header seq: 104004 +stamp: + secs: 1735696163 + nsecs: 689752102 +frame_id: "camera_color_optical_frame" +[INFO] [1735696164.139380]: received mask_obs with header seq: 350 +stamp: + secs: 1735696164 + nsecs: 140703678 +frame_id: '' and grasped_object_mask header seq: 104010 +stamp: + secs: 1735696163 + nsecs: 889593840 +frame_id: "camera_color_optical_frame" +[INFO] [1735696164.327778]: received mask_obs with header seq: 351 +stamp: + secs: 1735696164 + nsecs: 328962326 +frame_id: '' and grasped_object_mask header seq: 104016 +stamp: + secs: 1735696164 + nsecs: 89435339 +frame_id: "camera_color_optical_frame" +[INFO] [1735696164.527317]: received mask_obs with header seq: 352 +stamp: + secs: 1735696164 + nsecs: 528736352 +frame_id: '' and grasped_object_mask header seq: 104022 +stamp: + secs: 1735696164 + nsecs: 289276600 +frame_id: "camera_color_optical_frame" +[INFO] [1735696164.729142]: received mask_obs with header seq: 353 +stamp: + secs: 1735696164 + nsecs: 730477333 +frame_id: '' and grasped_object_mask header seq: 104028 +stamp: + secs: 1735696164 + nsecs: 489117384 +frame_id: "camera_color_optical_frame" +[INFO] [1735696164.928086]: received mask_obs with header seq: 354 +stamp: + secs: 1735696164 + nsecs: 929445743 +frame_id: '' and grasped_object_mask header seq: 104034 +stamp: + secs: 1735696164 + nsecs: 688959122 +frame_id: "camera_color_optical_frame" +[INFO] [1735696165.131841]: received mask_obs with header seq: 355 +stamp: + secs: 1735696165 + nsecs: 132869958 +frame_id: '' and grasped_object_mask header seq: 104040 +stamp: + secs: 1735696164 + nsecs: 888800144 +frame_id: "camera_color_optical_frame" +[INFO] [1735696165.329971]: received mask_obs with header seq: 356 +stamp: + secs: 1735696165 + nsecs: 331215620 +frame_id: '' and grasped_object_mask header seq: 104046 +stamp: + secs: 1735696165 + nsecs: 88641167 +frame_id: "camera_color_optical_frame" +[INFO] [1735696165.527873]: received mask_obs with header seq: 357 +stamp: + secs: 1735696165 + nsecs: 529164791 +frame_id: '' and grasped_object_mask header seq: 104052 +stamp: + secs: 1735696165 + nsecs: 288479805 +frame_id: "camera_color_optical_frame" +[INFO] [1735696165.728489]: received mask_obs with header seq: 358 +stamp: + secs: 1735696165 + nsecs: 729612827 +frame_id: '' and grasped_object_mask header seq: 104058 +stamp: + secs: 1735696165 + nsecs: 488312721 +frame_id: "camera_color_optical_frame" +[INFO] [1735696165.928050]: received mask_obs with header seq: 359 +stamp: + secs: 1735696165 + nsecs: 929485559 +frame_id: '' and grasped_object_mask header seq: 104064 +stamp: + secs: 1735696165 + nsecs: 688145399 +frame_id: "camera_color_optical_frame" +[INFO] [1735696166.128376]: received mask_obs with header seq: 360 +stamp: + secs: 1735696166 + nsecs: 129780769 +frame_id: '' and grasped_object_mask header seq: 104070 +stamp: + secs: 1735696165 + nsecs: 887979031 +frame_id: "camera_color_optical_frame" +[INFO] [1735696166.328545]: received mask_obs with header seq: 361 +stamp: + secs: 1735696166 + nsecs: 329699039 +frame_id: '' and grasped_object_mask header seq: 104076 +stamp: + secs: 1735696166 + nsecs: 87811232 +frame_id: "camera_color_optical_frame" +[INFO] [1735696166.547477]: received mask_obs with header seq: 362 +stamp: + secs: 1735696166 + nsecs: 548486232 +frame_id: '' and grasped_object_mask header seq: 104082 +stamp: + secs: 1735696166 + nsecs: 287644625 +frame_id: "camera_color_optical_frame" +[INFO] [1735696166.745339]: received mask_obs with header seq: 363 +stamp: + secs: 1735696166 + nsecs: 746740341 +frame_id: '' and grasped_object_mask header seq: 104089 +stamp: + secs: 1735696166 + nsecs: 520783901 +frame_id: "camera_color_optical_frame" +[INFO] [1735696166.936013]: received mask_obs with header seq: 364 +stamp: + secs: 1735696166 + nsecs: 936854839 +frame_id: '' and grasped_object_mask header seq: 104094 +stamp: + secs: 1735696166 + nsecs: 687311172 +frame_id: "camera_color_optical_frame" +[INFO] [1735696167.129656]: received mask_obs with header seq: 365 +stamp: + secs: 1735696167 + nsecs: 130573749 +frame_id: '' and grasped_object_mask header seq: 104100 +stamp: + secs: 1735696166 + nsecs: 887144089 +frame_id: "camera_color_optical_frame" +[INFO] [1735696167.326533]: received mask_obs with header seq: 366 +stamp: + secs: 1735696167 + nsecs: 327975749 +frame_id: '' and grasped_object_mask header seq: 104106 +stamp: + secs: 1735696167 + nsecs: 86977720 +frame_id: "camera_color_optical_frame" +[INFO] [1735696167.525847]: received mask_obs with header seq: 367 +stamp: + secs: 1735696167 + nsecs: 527077674 +frame_id: '' and grasped_object_mask header seq: 104112 +stamp: + secs: 1735696167 + nsecs: 286810875 +frame_id: "camera_color_optical_frame" +[INFO] [1735696167.727736]: received mask_obs with header seq: 368 +stamp: + secs: 1735696167 + nsecs: 729180335 +frame_id: '' and grasped_object_mask header seq: 104118 +stamp: + secs: 1735696167 + nsecs: 486646175 +frame_id: "camera_color_optical_frame" +[INFO] [1735696167.926817]: received mask_obs with header seq: 369 +stamp: + secs: 1735696167 + nsecs: 927954673 +frame_id: '' and grasped_object_mask header seq: 104124 +stamp: + secs: 1735696167 + nsecs: 686481476 +frame_id: "camera_color_optical_frame" +[INFO] [1735696168.147189]: received mask_obs with header seq: 370 +stamp: + secs: 1735696168 + nsecs: 148213386 +frame_id: '' and grasped_object_mask header seq: 104130 +stamp: + secs: 1735696167 + nsecs: 886316299 +frame_id: "camera_color_optical_frame" +[INFO] [1735696168.347502]: received mask_obs with header seq: 371 +stamp: + secs: 1735696168 + nsecs: 348670482 +frame_id: '' and grasped_object_mask header seq: 104137 +stamp: + secs: 1735696168 + nsecs: 119457722 +frame_id: "camera_color_optical_frame" +[INFO] [1735696168.540833]: received mask_obs with header seq: 372 +stamp: + secs: 1735696168 + nsecs: 542140245 +frame_id: '' and grasped_object_mask header seq: 104143 +stamp: + secs: 1735696168 + nsecs: 319294214 +frame_id: "camera_color_optical_frame" +[INFO] [1735696168.731888]: received mask_obs with header seq: 373 +stamp: + secs: 1735696168 + nsecs: 733276844 +frame_id: '' and grasped_object_mask header seq: 104148 +stamp: + secs: 1735696168 + nsecs: 485827923 +frame_id: "camera_color_optical_frame" +[INFO] [1735696168.926176]: received mask_obs with header seq: 374 +stamp: + secs: 1735696168 + nsecs: 927583217 +frame_id: '' and grasped_object_mask header seq: 104154 +stamp: + secs: 1735696168 + nsecs: 685667753 +frame_id: "camera_color_optical_frame" +[INFO] [1735696169.127943]: received mask_obs with header seq: 375 +stamp: + secs: 1735696169 + nsecs: 129385948 +frame_id: '' and grasped_object_mask header seq: 104160 +stamp: + secs: 1735696168 + nsecs: 885507822 +frame_id: "camera_color_optical_frame" +[INFO] [1735696169.326875]: received mask_obs with header seq: 376 +stamp: + secs: 1735696169 + nsecs: 328081846 +frame_id: '' and grasped_object_mask header seq: 104166 +stamp: + secs: 1735696169 + nsecs: 85347652 +frame_id: "camera_color_optical_frame" +[INFO] [1735696169.530461]: received mask_obs with header seq: 377 +stamp: + secs: 1735696169 + nsecs: 531918525 +frame_id: '' and grasped_object_mask header seq: 104172 +stamp: + secs: 1735696169 + nsecs: 285187244 +frame_id: "camera_color_optical_frame" +[INFO] [1735696169.729346]: received mask_obs with header seq: 378 +stamp: + secs: 1735696169 + nsecs: 730746030 +frame_id: '' and grasped_object_mask header seq: 104178 +stamp: + secs: 1735696169 + nsecs: 485024214 +frame_id: "camera_color_optical_frame" +[INFO] [1735696169.923665]: received mask_obs with header seq: 379 +stamp: + secs: 1735696169 + nsecs: 925096988 +frame_id: '' and grasped_object_mask header seq: 104184 +stamp: + secs: 1735696169 + nsecs: 684862137 +frame_id: "camera_color_optical_frame" +[INFO] [1735696170.127851]: received mask_obs with header seq: 380 +stamp: + secs: 1735696170 + nsecs: 129264831 +frame_id: '' and grasped_object_mask header seq: 104190 +stamp: + secs: 1735696169 + nsecs: 884698629 +frame_id: "camera_color_optical_frame" +[INFO] [1735696170.324980]: received mask_obs with header seq: 381 +stamp: + secs: 1735696170 + nsecs: 326422452 +frame_id: '' and grasped_object_mask header seq: 104196 +stamp: + secs: 1735696170 + nsecs: 84535122 +frame_id: "camera_color_optical_frame" +[INFO] [1735696170.543254]: received mask_obs with header seq: 382 +stamp: + secs: 1735696170 + nsecs: 544762134 +frame_id: '' and grasped_object_mask header seq: 104202 +stamp: + secs: 1735696170 + nsecs: 284372091 +frame_id: "camera_color_optical_frame" +[INFO] [1735696170.744755]: received mask_obs with header seq: 383 +stamp: + secs: 1735696170 + nsecs: 745831727 +frame_id: '' and grasped_object_mask header seq: 104209 +stamp: + secs: 1735696170 + nsecs: 517518759 +frame_id: "camera_color_optical_frame" +[INFO] [1735696170.942752]: received mask_obs with header seq: 384 +stamp: + secs: 1735696170 + nsecs: 944214344 +frame_id: '' and grasped_object_mask header seq: 104215 +stamp: + secs: 1735696170 + nsecs: 717358351 +frame_id: "camera_color_optical_frame" +[INFO] [1735696171.140268]: received mask_obs with header seq: 385 +stamp: + secs: 1735696171 + nsecs: 141720533 +frame_id: '' and grasped_object_mask header seq: 104221 +stamp: + secs: 1735696170 + nsecs: 917198181 +frame_id: "camera_color_optical_frame" +[INFO] [1735696171.329155]: received mask_obs with header seq: 386 +stamp: + secs: 1735696171 + nsecs: 330569744 +frame_id: '' and grasped_object_mask header seq: 104226 +stamp: + secs: 1735696171 + nsecs: 83731890 +frame_id: "camera_color_optical_frame" +[INFO] [1735696171.529047]: received mask_obs with header seq: 387 +stamp: + secs: 1735696171 + nsecs: 530578136 +frame_id: '' and grasped_object_mask header seq: 104232 +stamp: + secs: 1735696171 + nsecs: 283570528 +frame_id: "camera_color_optical_frame" +[INFO] [1735696171.736157]: received mask_obs with header seq: 388 +stamp: + secs: 1735696171 + nsecs: 737523317 +frame_id: '' and grasped_object_mask header seq: 104238 +stamp: + secs: 1735696171 + nsecs: 483404398 +frame_id: "camera_color_optical_frame" +[INFO] [1735696171.931519]: received mask_obs with header seq: 389 +stamp: + secs: 1735696171 + nsecs: 932067155 +frame_id: '' and grasped_object_mask header seq: 104245 +stamp: + secs: 1735696171 + nsecs: 716543913 +frame_id: "camera_color_optical_frame" +[INFO] [1735696172.131087]: received mask_obs with header seq: 390 +stamp: + secs: 1735696172 + nsecs: 131998062 +frame_id: '' and grasped_object_mask header seq: 104250 +stamp: + secs: 1735696171 + nsecs: 883070946 +frame_id: "camera_color_optical_frame" +[INFO] [1735696172.331788]: received mask_obs with header seq: 391 +stamp: + secs: 1735696172 + nsecs: 332014799 +frame_id: '' and grasped_object_mask header seq: 104256 +stamp: + secs: 1735696172 + nsecs: 82905054 +frame_id: "camera_color_optical_frame" +[INFO] [1735696172.527780]: received mask_obs with header seq: 392 +stamp: + secs: 1735696172 + nsecs: 529042482 +frame_id: '' and grasped_object_mask header seq: 104263 +stamp: + secs: 1735696172 + nsecs: 316044807 +frame_id: "camera_color_optical_frame" +[INFO] [1735696172.735658]: received mask_obs with header seq: 393 +stamp: + secs: 1735696172 + nsecs: 737053155 +frame_id: '' and grasped_object_mask header seq: 104268 +stamp: + secs: 1735696172 + nsecs: 482577801 +frame_id: "camera_color_optical_frame" +[INFO] [1735696172.930799]: received mask_obs with header seq: 394 +stamp: + secs: 1735696172 + nsecs: 931374549 +frame_id: '' and grasped_object_mask header seq: 104274 +stamp: + secs: 1735696172 + nsecs: 682416677 +frame_id: "camera_color_optical_frame" +[INFO] [1735696173.134822]: received mask_obs with header seq: 395 +stamp: + secs: 1735696173 + nsecs: 135623216 +frame_id: '' and grasped_object_mask header seq: 104281 +stamp: + secs: 1735696172 + nsecs: 915563107 +frame_id: "camera_color_optical_frame" +[INFO] [1735696173.331080]: received mask_obs with header seq: 396 +stamp: + secs: 1735696173 + nsecs: 332577943 +frame_id: '' and grasped_object_mask header seq: 104287 +stamp: + secs: 1735696173 + nsecs: 115401983 +frame_id: "camera_color_optical_frame" +[INFO] [1735696173.535306]: received mask_obs with header seq: 397 +stamp: + secs: 1735696173 + nsecs: 535696029 +frame_id: '' and grasped_object_mask header seq: 104293 +stamp: + secs: 1735696173 + nsecs: 315242290 +frame_id: "camera_color_optical_frame" +[INFO] [1735696173.740865]: received mask_obs with header seq: 398 +stamp: + secs: 1735696173 + nsecs: 741396188 +frame_id: '' and grasped_object_mask header seq: 104299 +stamp: + secs: 1735696173 + nsecs: 515084743 +frame_id: "camera_color_optical_frame" +[INFO] [1735696173.776890]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735696173.945488]: received mask_obs with header seq: 399 +stamp: + secs: 1735696173 + nsecs: 941737651 +frame_id: '' and grasped_object_mask header seq: 104305 +stamp: + secs: 1735696173 + nsecs: 714927435 +frame_id: "camera_color_optical_frame" +[INFO] [1735696174.545181]: Stored episode 3. +[INFO] [1735696174.555009]: Saving video... +[INFO] [1735696174.754203]: Video saved! +Starting episode 3 +[INFO] [1735696174.804212]: resetting environment +[INFO] [1735696174.820569]: cleared current plan +[INFO] [1735696174.823704]: moving to home +[INFO] [1735696178.434582]: reached home +[INFO] [1735696178.437601]: reset action history +[INFO] [1735696181.221801]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735696186.002688]: resetting environment +[INFO] [1735696186.021062]: cleared current plan +[INFO] [1735696186.024656]: moving to home +[INFO] [1735696187.031878]: reached home +[INFO] [1735696187.033326]: reset action history +[INFO] [1735696189.820255]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/episode_rosbags/episode_3_2024-12-31-20-49-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1737022.log +started roslaunch server http://158.130.50.37:41719/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1737970] +process[print_text-11]: started with pid [1737971] +process[rosbag_record-12]: started with pid [1737995] +started bagging! +[INFO] [1735696190.542618]: received mask_obs with header seq: 400 +stamp: + secs: 1735696190 + nsecs: 543778419 +frame_id: '' and grasped_object_mask header seq: 104804 +stamp: + secs: 1735696190 + nsecs: 329763889 +frame_id: "camera_color_optical_frame" +[INFO] [1735696190.747818]: received mask_obs with header seq: 401 +stamp: + secs: 1735696190 + nsecs: 749050140 +frame_id: '' and grasped_object_mask header seq: 104810 +stamp: + secs: 1735696190 + nsecs: 529381990 +frame_id: "camera_color_optical_frame" +[INFO] [1735696190.947730]: received mask_obs with header seq: 402 +stamp: + secs: 1735696190 + nsecs: 949217796 +frame_id: '' and grasped_object_mask header seq: 104816 +stamp: + secs: 1735696190 + nsecs: 728987455 +frame_id: "camera_color_optical_frame" +[INFO] [1735696191.150970]: received mask_obs with header seq: 403 +stamp: + secs: 1735696191 + nsecs: 152496337 +frame_id: '' and grasped_object_mask header seq: 104822 +stamp: + secs: 1735696190 + nsecs: 928581238 +frame_id: "camera_color_optical_frame" +[INFO] [1735696191.359534]: received mask_obs with header seq: 404 +stamp: + secs: 1735696191 + nsecs: 360915184 +frame_id: '' and grasped_object_mask header seq: 104828 +stamp: + secs: 1735696191 + nsecs: 128163576 +frame_id: "camera_color_optical_frame" +[INFO] [1735696191.564307]: received mask_obs with header seq: 405 +stamp: + secs: 1735696191 + nsecs: 565000534 +frame_id: '' and grasped_object_mask header seq: 104834 +stamp: + secs: 1735696191 + nsecs: 327871561 +frame_id: "camera_color_optical_frame" +[INFO] [1735696191.771049]: received mask_obs with header seq: 406 +stamp: + secs: 1735696191 + nsecs: 772578716 +frame_id: '' and grasped_object_mask header seq: 104840 +stamp: + secs: 1735696191 + nsecs: 528779745 +frame_id: "camera_color_optical_frame" +[INFO] [1735696191.971755]: received mask_obs with header seq: 407 +stamp: + secs: 1735696191 + nsecs: 973296403 +frame_id: '' and grasped_object_mask header seq: 104846 +stamp: + secs: 1735696191 + nsecs: 729776144 +frame_id: "camera_color_optical_frame" +[INFO] [1735696192.171719]: received mask_obs with header seq: 408 +stamp: + secs: 1735696192 + nsecs: 172915220 +frame_id: '' and grasped_object_mask header seq: 104852 +stamp: + secs: 1735696191 + nsecs: 930858374 +frame_id: "camera_color_optical_frame" +[INFO] [1735696192.373042]: received mask_obs with header seq: 409 +stamp: + secs: 1735696192 + nsecs: 374402523 +frame_id: '' and grasped_object_mask header seq: 104858 +stamp: + secs: 1735696192 + nsecs: 132028103 +frame_id: "camera_color_optical_frame" +[INFO] [1735696192.574205]: received mask_obs with header seq: 410 +stamp: + secs: 1735696192 + nsecs: 575181961 +frame_id: '' and grasped_object_mask header seq: 104864 +stamp: + secs: 1735696192 + nsecs: 333135605 +frame_id: "camera_color_optical_frame" +[INFO] [1735696192.771404]: received mask_obs with header seq: 411 +stamp: + secs: 1735696192 + nsecs: 772784948 +frame_id: '' and grasped_object_mask header seq: 104870 +stamp: + secs: 1735696192 + nsecs: 532987118 +frame_id: "camera_color_optical_frame" +[INFO] [1735696192.974090]: received mask_obs with header seq: 412 +stamp: + secs: 1735696192 + nsecs: 974897384 +frame_id: '' and grasped_object_mask header seq: 104876 +stamp: + secs: 1735696192 + nsecs: 732839346 +frame_id: "camera_color_optical_frame" +[INFO] [1735696193.172350]: received mask_obs with header seq: 413 +stamp: + secs: 1735696193 + nsecs: 173546552 +frame_id: '' and grasped_object_mask header seq: 104882 +stamp: + secs: 1735696192 + nsecs: 932692051 +frame_id: "camera_color_optical_frame" +[INFO] [1735696193.372449]: received mask_obs with header seq: 414 +stamp: + secs: 1735696193 + nsecs: 373816728 +frame_id: '' and grasped_object_mask header seq: 104888 +stamp: + secs: 1735696193 + nsecs: 132546663 +frame_id: "camera_color_optical_frame" +[INFO] [1735696193.571353]: received mask_obs with header seq: 415 +stamp: + secs: 1735696193 + nsecs: 572920083 +frame_id: '' and grasped_object_mask header seq: 104894 +stamp: + secs: 1735696193 + nsecs: 332399607 +frame_id: "camera_color_optical_frame" +[INFO] [1735696193.770588]: received mask_obs with header seq: 416 +stamp: + secs: 1735696193 + nsecs: 772114753 +frame_id: '' and grasped_object_mask header seq: 104900 +stamp: + secs: 1735696193 + nsecs: 532244682 +frame_id: "camera_color_optical_frame" +[INFO] [1735696193.971399]: received mask_obs with header seq: 417 +stamp: + secs: 1735696193 + nsecs: 972819805 +frame_id: '' and grasped_object_mask header seq: 104906 +stamp: + secs: 1735696193 + nsecs: 732089520 +frame_id: "camera_color_optical_frame" +[INFO] [1735696194.182374]: received mask_obs with header seq: 418 +stamp: + secs: 1735696194 + nsecs: 183845520 +frame_id: '' and grasped_object_mask header seq: 104912 +stamp: + secs: 1735696193 + nsecs: 931935072 +frame_id: "camera_color_optical_frame" +[INFO] [1735696194.396421]: received mask_obs with header seq: 419 +stamp: + secs: 1735696194 + nsecs: 397757530 +frame_id: '' and grasped_object_mask header seq: 104918 +stamp: + secs: 1735696194 + nsecs: 131780624 +frame_id: "camera_color_optical_frame" +[INFO] [1735696194.601589]: received mask_obs with header seq: 420 +stamp: + secs: 1735696194 + nsecs: 602761030 +frame_id: '' and grasped_object_mask header seq: 104925 +stamp: + secs: 1735696194 + nsecs: 364934921 +frame_id: "camera_color_optical_frame" +[INFO] [1735696194.798525]: received mask_obs with header seq: 421 +stamp: + secs: 1735696194 + nsecs: 800046682 +frame_id: '' and grasped_object_mask header seq: 104931 +stamp: + secs: 1735696194 + nsecs: 564781189 +frame_id: "camera_color_optical_frame" +[INFO] [1735696195.001396]: received mask_obs with header seq: 422 +stamp: + secs: 1735696195 + nsecs: 2875089 +frame_id: '' and grasped_object_mask header seq: 104937 +stamp: + secs: 1735696194 + nsecs: 764628172 +frame_id: "camera_color_optical_frame" +[INFO] [1735696195.199517]: received mask_obs with header seq: 423 +stamp: + secs: 1735696195 + nsecs: 201041698 +frame_id: '' and grasped_object_mask header seq: 104943 +stamp: + secs: 1735696194 + nsecs: 964476585 +frame_id: "camera_color_optical_frame" +[INFO] [1735696195.399429]: received mask_obs with header seq: 424 +stamp: + secs: 1735696195 + nsecs: 400838613 +frame_id: '' and grasped_object_mask header seq: 104949 +stamp: + secs: 1735696195 + nsecs: 164324045 +frame_id: "camera_color_optical_frame" +[INFO] [1735696195.597947]: received mask_obs with header seq: 425 +stamp: + secs: 1735696195 + nsecs: 599433422 +frame_id: '' and grasped_object_mask header seq: 104955 +stamp: + secs: 1735696195 + nsecs: 364169598 +frame_id: "camera_color_optical_frame" +[INFO] [1735696195.796051]: received mask_obs with header seq: 426 +stamp: + secs: 1735696195 + nsecs: 797581911 +frame_id: '' and grasped_object_mask header seq: 104961 +stamp: + secs: 1735696195 + nsecs: 564008713 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.001436]: received mask_obs with header seq: 427 +stamp: + secs: 1735696196 + nsecs: 2839803 +frame_id: '' and grasped_object_mask header seq: 104967 +stamp: + secs: 1735696195 + nsecs: 763847828 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.200850]: received mask_obs with header seq: 428 +stamp: + secs: 1735696196 + nsecs: 202284336 +frame_id: '' and grasped_object_mask header seq: 104973 +stamp: + secs: 1735696195 + nsecs: 963686705 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.398770]: received mask_obs with header seq: 429 +stamp: + secs: 1735696196 + nsecs: 399369716 +frame_id: '' and grasped_object_mask header seq: 104979 +stamp: + secs: 1735696196 + nsecs: 163526535 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.600793]: received mask_obs with header seq: 430 +stamp: + secs: 1735696196 + nsecs: 602343320 +frame_id: '' and grasped_object_mask header seq: 104985 +stamp: + secs: 1735696196 + nsecs: 363365889 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.794501]: received mask_obs with header seq: 431 +stamp: + secs: 1735696196 + nsecs: 796006202 +frame_id: '' and grasped_object_mask header seq: 104991 +stamp: + secs: 1735696196 + nsecs: 563206196 +frame_id: "camera_color_optical_frame" +[INFO] [1735696196.995135]: received mask_obs with header seq: 432 +stamp: + secs: 1735696196 + nsecs: 996651172 +frame_id: '' and grasped_object_mask header seq: 104997 +stamp: + secs: 1735696196 + nsecs: 763046503 +frame_id: "camera_color_optical_frame" +[INFO] [1735696197.188537]: received mask_obs with header seq: 433 +stamp: + secs: 1735696197 + nsecs: 189388990 +frame_id: '' and grasped_object_mask header seq: 105002 +stamp: + secs: 1735696196 + nsecs: 929580212 +frame_id: "camera_color_optical_frame" +[INFO] [1735696197.379267]: received mask_obs with header seq: 434 +stamp: + secs: 1735696197 + nsecs: 380764722 +frame_id: '' and grasped_object_mask header seq: 105008 +stamp: + secs: 1735696197 + nsecs: 129420757 +frame_id: "camera_color_optical_frame" +[INFO] [1735696197.576076]: received mask_obs with header seq: 435 +stamp: + secs: 1735696197 + nsecs: 577567815 +frame_id: '' and grasped_object_mask header seq: 105014 +stamp: + secs: 1735696197 + nsecs: 329260349 +frame_id: "camera_color_optical_frame" +[INFO] [1735696197.774259]: received mask_obs with header seq: 436 +stamp: + secs: 1735696197 + nsecs: 775258779 +frame_id: '' and grasped_object_mask header seq: 105020 +stamp: + secs: 1735696197 + nsecs: 529096127 +frame_id: "camera_color_optical_frame" +[INFO] [1735696197.971690]: received mask_obs with header seq: 437 +stamp: + secs: 1735696197 + nsecs: 973166227 +frame_id: '' and grasped_object_mask header seq: 105026 +stamp: + secs: 1735696197 + nsecs: 728932142 +frame_id: "camera_color_optical_frame" +[INFO] [1735696198.170703]: received mask_obs with header seq: 438 +stamp: + secs: 1735696198 + nsecs: 172162532 +frame_id: '' and grasped_object_mask header seq: 105032 +stamp: + secs: 1735696197 + nsecs: 928768873 +frame_id: "camera_color_optical_frame" +[INFO] [1735696198.372799]: received mask_obs with header seq: 439 +stamp: + secs: 1735696198 + nsecs: 373675823 +frame_id: '' and grasped_object_mask header seq: 105038 +stamp: + secs: 1735696198 + nsecs: 128604412 +frame_id: "camera_color_optical_frame" +[INFO] [1735696198.574230]: received mask_obs with header seq: 440 +stamp: + secs: 1735696198 + nsecs: 574934959 +frame_id: '' and grasped_object_mask header seq: 105044 +stamp: + secs: 1735696198 + nsecs: 328440428 +frame_id: "camera_color_optical_frame" +[INFO] [1735696198.771807]: received mask_obs with header seq: 441 +stamp: + secs: 1735696198 + nsecs: 773255586 +frame_id: '' and grasped_object_mask header seq: 105050 +stamp: + secs: 1735696198 + nsecs: 528279543 +frame_id: "camera_color_optical_frame" +[INFO] [1735696198.972084]: received mask_obs with header seq: 442 +stamp: + secs: 1735696198 + nsecs: 973157644 +frame_id: '' and grasped_object_mask header seq: 105056 +stamp: + secs: 1735696198 + nsecs: 728118420 +frame_id: "camera_color_optical_frame" +[INFO] [1735696199.172439]: received mask_obs with header seq: 443 +stamp: + secs: 1735696199 + nsecs: 173894166 +frame_id: '' and grasped_object_mask header seq: 105062 +stamp: + secs: 1735696198 + nsecs: 927957296 +frame_id: "camera_color_optical_frame" +[INFO] [1735696199.371481]: received mask_obs with header seq: 444 +stamp: + secs: 1735696199 + nsecs: 372955799 +frame_id: '' and grasped_object_mask header seq: 105068 +stamp: + secs: 1735696199 + nsecs: 127795935 +frame_id: "camera_color_optical_frame" +[INFO] [1735696199.575154]: received mask_obs with header seq: 445 +stamp: + secs: 1735696199 + nsecs: 576423168 +frame_id: '' and grasped_object_mask header seq: 105074 +stamp: + secs: 1735696199 + nsecs: 327636003 +frame_id: "camera_color_optical_frame" +[INFO] [1735696199.772272]: received mask_obs with header seq: 446 +stamp: + secs: 1735696199 + nsecs: 773744344 +frame_id: '' and grasped_object_mask header seq: 105080 +stamp: + secs: 1735696199 + nsecs: 527480364 +frame_id: "camera_color_optical_frame" +[INFO] [1735696199.975777]: received mask_obs with header seq: 447 +stamp: + secs: 1735696199 + nsecs: 976433515 +frame_id: '' and grasped_object_mask header seq: 105086 +stamp: + secs: 1735696199 + nsecs: 727325678 +frame_id: "camera_color_optical_frame" +[INFO] [1735696200.183444]: received mask_obs with header seq: 448 +stamp: + secs: 1735696200 + nsecs: 184874534 +frame_id: '' and grasped_object_mask header seq: 105093 +stamp: + secs: 1735696199 + nsecs: 960479021 +frame_id: "camera_color_optical_frame" +[INFO] [1735696200.379954]: received mask_obs with header seq: 449 +stamp: + secs: 1735696200 + nsecs: 381491184 +frame_id: '' and grasped_object_mask header seq: 105099 +stamp: + secs: 1735696200 + nsecs: 160324574 +frame_id: "camera_color_optical_frame" +[INFO] [1735696200.581105]: received mask_obs with header seq: 450 +stamp: + secs: 1735696200 + nsecs: 582292795 +frame_id: '' and grasped_object_mask header seq: 105105 +stamp: + secs: 1735696200 + nsecs: 360167980 +frame_id: "camera_color_optical_frame" +[INFO] [1735696200.783081]: received mask_obs with header seq: 451 +stamp: + secs: 1735696200 + nsecs: 784612894 +frame_id: '' and grasped_object_mask header seq: 105111 +stamp: + secs: 1735696200 + nsecs: 560003757 +frame_id: "camera_color_optical_frame" +[INFO] [1735696200.985528]: received mask_obs with header seq: 452 +stamp: + secs: 1735696200 + nsecs: 987082958 +frame_id: '' and grasped_object_mask header seq: 105117 +stamp: + secs: 1735696200 + nsecs: 759839058 +frame_id: "camera_color_optical_frame" +[INFO] [1735696201.191736]: received mask_obs with header seq: 453 +stamp: + secs: 1735696201 + nsecs: 193317890 +frame_id: '' and grasped_object_mask header seq: 105123 +stamp: + secs: 1735696200 + nsecs: 959674597 +frame_id: "camera_color_optical_frame" +[INFO] [1735696201.397252]: received mask_obs with header seq: 454 +stamp: + secs: 1735696201 + nsecs: 398723602 +frame_id: '' and grasped_object_mask header seq: 105129 +stamp: + secs: 1735696201 + nsecs: 159510612 +frame_id: "camera_color_optical_frame" +[INFO] [1735696201.595426]: received mask_obs with header seq: 455 +stamp: + secs: 1735696201 + nsecs: 596949815 +frame_id: '' and grasped_object_mask header seq: 105135 +stamp: + secs: 1735696201 + nsecs: 359345675 +frame_id: "camera_color_optical_frame" +[INFO] [1735696201.800358]: received mask_obs with header seq: 456 +stamp: + secs: 1735696201 + nsecs: 801898956 +frame_id: '' and grasped_object_mask header seq: 105141 +stamp: + secs: 1735696201 + nsecs: 559182644 +frame_id: "camera_color_optical_frame" +[INFO] [1735696201.995493]: received mask_obs with header seq: 457 +stamp: + secs: 1735696201 + nsecs: 996813774 +frame_id: '' and grasped_object_mask header seq: 105147 +stamp: + secs: 1735696201 + nsecs: 759019375 +frame_id: "camera_color_optical_frame" +[INFO] [1735696202.189080]: received mask_obs with header seq: 458 +stamp: + secs: 1735696202 + nsecs: 190620660 +frame_id: '' and grasped_object_mask header seq: 105153 +stamp: + secs: 1735696201 + nsecs: 958855629 +frame_id: "camera_color_optical_frame" +[INFO] [1735696202.384946]: received mask_obs with header seq: 459 +stamp: + secs: 1735696202 + nsecs: 386047124 +frame_id: '' and grasped_object_mask header seq: 105159 +stamp: + secs: 1735696202 + nsecs: 158691883 +frame_id: "camera_color_optical_frame" +[INFO] [1735696202.585314]: received mask_obs with header seq: 460 +stamp: + secs: 1735696202 + nsecs: 586872339 +frame_id: '' and grasped_object_mask header seq: 105165 +stamp: + secs: 1735696202 + nsecs: 358529329 +frame_id: "camera_color_optical_frame" +[INFO] [1735696202.781616]: received mask_obs with header seq: 461 +stamp: + secs: 1735696202 + nsecs: 783104658 +frame_id: '' and grasped_object_mask header seq: 105171 +stamp: + secs: 1735696202 + nsecs: 558367252 +frame_id: "camera_color_optical_frame" +[INFO] [1735696202.978740]: received mask_obs with header seq: 462 +stamp: + secs: 1735696202 + nsecs: 980190753 +frame_id: '' and grasped_object_mask header seq: 105177 +stamp: + secs: 1735696202 + nsecs: 758204937 +frame_id: "camera_color_optical_frame" +[INFO] [1735696203.183735]: received mask_obs with header seq: 463 +stamp: + secs: 1735696203 + nsecs: 185309410 +frame_id: '' and grasped_object_mask header seq: 105183 +stamp: + secs: 1735696202 + nsecs: 958042383 +frame_id: "camera_color_optical_frame" +[INFO] [1735696203.379399]: received mask_obs with header seq: 464 +stamp: + secs: 1735696203 + nsecs: 380691528 +frame_id: '' and grasped_object_mask header seq: 105189 +stamp: + secs: 1735696203 + nsecs: 157879829 +frame_id: "camera_color_optical_frame" +[INFO] [1735696203.584490]: received mask_obs with header seq: 465 +stamp: + secs: 1735696203 + nsecs: 586075067 +frame_id: '' and grasped_object_mask header seq: 105195 +stamp: + secs: 1735696203 + nsecs: 357717514 +frame_id: "camera_color_optical_frame" +[INFO] [1735696203.792158]: received mask_obs with header seq: 466 +stamp: + secs: 1735696203 + nsecs: 793278455 +frame_id: '' and grasped_object_mask header seq: 105201 +stamp: + secs: 1735696203 + nsecs: 557557344 +frame_id: "camera_color_optical_frame" +[INFO] [1735696203.993155]: received mask_obs with header seq: 467 +stamp: + secs: 1735696203 + nsecs: 994714260 +frame_id: '' and grasped_object_mask header seq: 105207 +stamp: + secs: 1735696203 + nsecs: 757396221 +frame_id: "camera_color_optical_frame" +[INFO] [1735696204.182910]: received mask_obs with header seq: 468 +stamp: + secs: 1735696204 + nsecs: 184525489 +frame_id: '' and grasped_object_mask header seq: 105213 +stamp: + secs: 1735696203 + nsecs: 957234859 +frame_id: "camera_color_optical_frame" +[INFO] [1735696204.384005]: received mask_obs with header seq: 469 +stamp: + secs: 1735696204 + nsecs: 385600090 +frame_id: '' and grasped_object_mask header seq: 105219 +stamp: + secs: 1735696204 + nsecs: 157073498 +frame_id: "camera_color_optical_frame" +[INFO] [1735696204.582317]: received mask_obs with header seq: 470 +stamp: + secs: 1735696204 + nsecs: 583585739 +frame_id: '' and grasped_object_mask header seq: 105225 +stamp: + secs: 1735696204 + nsecs: 356910944 +frame_id: "camera_color_optical_frame" +[INFO] [1735696204.782217]: received mask_obs with header seq: 471 +stamp: + secs: 1735696204 + nsecs: 783555507 +frame_id: '' and grasped_object_mask header seq: 105231 +stamp: + secs: 1735696204 + nsecs: 556744099 +frame_id: "camera_color_optical_frame" +[INFO] [1735696204.980890]: received mask_obs with header seq: 472 +stamp: + secs: 1735696204 + nsecs: 982503175 +frame_id: '' and grasped_object_mask header seq: 105237 +stamp: + secs: 1735696204 + nsecs: 756578445 +frame_id: "camera_color_optical_frame" +[INFO] [1735696205.179902]: received mask_obs with header seq: 473 +stamp: + secs: 1735696205 + nsecs: 181475400 +frame_id: '' and grasped_object_mask header seq: 105243 +stamp: + secs: 1735696204 + nsecs: 956411123 +frame_id: "camera_color_optical_frame" +[INFO] [1735696205.378009]: received mask_obs with header seq: 474 +stamp: + secs: 1735696205 + nsecs: 379630804 +frame_id: '' and grasped_object_mask header seq: 105249 +stamp: + secs: 1735696205 + nsecs: 156243801 +frame_id: "camera_color_optical_frame" +[INFO] [1735696205.580895]: received mask_obs with header seq: 475 +stamp: + secs: 1735696205 + nsecs: 582351446 +frame_id: '' and grasped_object_mask header seq: 105255 +stamp: + secs: 1735696205 + nsecs: 356076002 +frame_id: "camera_color_optical_frame" +[INFO] [1735696205.778274]: received mask_obs with header seq: 476 +stamp: + secs: 1735696205 + nsecs: 779584169 +frame_id: '' and grasped_object_mask header seq: 105261 +stamp: + secs: 1735696205 + nsecs: 555906296 +frame_id: "camera_color_optical_frame" +[INFO] [1735696205.985092]: received mask_obs with header seq: 477 +stamp: + secs: 1735696205 + nsecs: 986723899 +frame_id: '' and grasped_object_mask header seq: 105267 +stamp: + secs: 1735696205 + nsecs: 755736351 +frame_id: "camera_color_optical_frame" +[INFO] [1735696206.186190]: received mask_obs with header seq: 478 +stamp: + secs: 1735696206 + nsecs: 187721014 +frame_id: '' and grasped_object_mask header seq: 105273 +stamp: + secs: 1735696205 + nsecs: 955566883 +frame_id: "camera_color_optical_frame" +[INFO] [1735696206.381063]: received mask_obs with header seq: 479 +stamp: + secs: 1735696206 + nsecs: 382473468 +frame_id: '' and grasped_object_mask header seq: 105279 +stamp: + secs: 1735696206 + nsecs: 155395746 +frame_id: "camera_color_optical_frame" +[INFO] [1735696206.581299]: received mask_obs with header seq: 480 +stamp: + secs: 1735696206 + nsecs: 582844018 +frame_id: '' and grasped_object_mask header seq: 105285 +stamp: + secs: 1735696206 + nsecs: 355224848 +frame_id: "camera_color_optical_frame" +[INFO] [1735696206.781067]: received mask_obs with header seq: 481 +stamp: + secs: 1735696206 + nsecs: 782700777 +frame_id: '' and grasped_object_mask header seq: 105291 +stamp: + secs: 1735696206 + nsecs: 555055380 +frame_id: "camera_color_optical_frame" +[INFO] [1735696206.980652]: received mask_obs with header seq: 482 +stamp: + secs: 1735696206 + nsecs: 981922388 +frame_id: '' and grasped_object_mask header seq: 105297 +stamp: + secs: 1735696206 + nsecs: 754885435 +frame_id: "camera_color_optical_frame" +[INFO] [1735696207.179136]: received mask_obs with header seq: 483 +stamp: + secs: 1735696207 + nsecs: 180759906 +frame_id: '' and grasped_object_mask header seq: 105303 +stamp: + secs: 1735696206 + nsecs: 954715490 +frame_id: "camera_color_optical_frame" +[INFO] [1735696207.379662]: received mask_obs with header seq: 484 +stamp: + secs: 1735696207 + nsecs: 381291151 +frame_id: '' and grasped_object_mask header seq: 105309 +stamp: + secs: 1735696207 + nsecs: 154546261 +frame_id: "camera_color_optical_frame" +[INFO] [1735696207.581177]: received mask_obs with header seq: 485 +stamp: + secs: 1735696207 + nsecs: 582697153 +frame_id: '' and grasped_object_mask header seq: 105315 +stamp: + secs: 1735696207 + nsecs: 354377985 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -100835461 for nearest: 1735696207453383546 - target: 1735696207554219007 at idx 218 +[INFO] [1735696207.781374]: received mask_obs with header seq: 486 +stamp: + secs: 1735696207 + nsecs: 782941341 +frame_id: '' and grasped_object_mask header seq: 105321 +stamp: + secs: 1735696207 + nsecs: 554219007 +frame_id: "camera_color_optical_frame" +[INFO] [1735696207.981071]: received mask_obs with header seq: 487 +stamp: + secs: 1735696207 + nsecs: 982467412 +frame_id: '' and grasped_object_mask header seq: 105327 +stamp: + secs: 1735696207 + nsecs: 754060030 +frame_id: "camera_color_optical_frame" +[INFO] [1735696208.180838]: received mask_obs with header seq: 488 +stamp: + secs: 1735696208 + nsecs: 181925773 +frame_id: '' and grasped_object_mask header seq: 105333 +stamp: + secs: 1735696207 + nsecs: 953901291 +frame_id: "camera_color_optical_frame" +[INFO] [1735696208.378071]: received mask_obs with header seq: 489 +stamp: + secs: 1735696208 + nsecs: 379658699 +frame_id: '' and grasped_object_mask header seq: 105339 +stamp: + secs: 1735696208 + nsecs: 153742790 +frame_id: "camera_color_optical_frame" +[INFO] [1735696208.581964]: received mask_obs with header seq: 490 +stamp: + secs: 1735696208 + nsecs: 583481550 +frame_id: '' and grasped_object_mask header seq: 105345 +stamp: + secs: 1735696208 + nsecs: 353583813 +frame_id: "camera_color_optical_frame" +[INFO] [1735696208.777087]: received mask_obs with header seq: 491 +stamp: + secs: 1735696208 + nsecs: 778397083 +frame_id: '' and grasped_object_mask header seq: 105351 +stamp: + secs: 1735696208 + nsecs: 553424835 +frame_id: "camera_color_optical_frame" +[INFO] [1735696208.977929]: received mask_obs with header seq: 492 +stamp: + secs: 1735696208 + nsecs: 979430913 +frame_id: '' and grasped_object_mask header seq: 105357 +stamp: + secs: 1735696208 + nsecs: 753265142 +frame_id: "camera_color_optical_frame" +[INFO] [1735696209.179479]: received mask_obs with header seq: 493 +stamp: + secs: 1735696209 + nsecs: 181088924 +frame_id: '' and grasped_object_mask header seq: 105363 +stamp: + secs: 1735696208 + nsecs: 953105450 +frame_id: "camera_color_optical_frame" +[INFO] [1735696209.376448]: received mask_obs with header seq: 494 +stamp: + secs: 1735696209 + nsecs: 378050088 +frame_id: '' and grasped_object_mask header seq: 105369 +stamp: + secs: 1735696209 + nsecs: 152945995 +frame_id: "camera_color_optical_frame" +[INFO] [1735696209.577962]: received mask_obs with header seq: 495 +stamp: + secs: 1735696209 + nsecs: 579564332 +frame_id: '' and grasped_object_mask header seq: 105375 +stamp: + secs: 1735696209 + nsecs: 352786541 +frame_id: "camera_color_optical_frame" +[INFO] [1735696209.777778]: received mask_obs with header seq: 496 +stamp: + secs: 1735696209 + nsecs: 779278755 +frame_id: '' and grasped_object_mask header seq: 105381 +stamp: + secs: 1735696209 + nsecs: 552626848 +frame_id: "camera_color_optical_frame" +[INFO] [1735696209.979635]: received mask_obs with header seq: 497 +stamp: + secs: 1735696209 + nsecs: 980951786 +frame_id: '' and grasped_object_mask header seq: 105387 +stamp: + secs: 1735696209 + nsecs: 752468586 +frame_id: "camera_color_optical_frame" +[INFO] [1735696210.177846]: received mask_obs with header seq: 498 +stamp: + secs: 1735696210 + nsecs: 179472446 +frame_id: '' and grasped_object_mask header seq: 105393 +stamp: + secs: 1735696209 + nsecs: 952309132 +frame_id: "camera_color_optical_frame" +[INFO] [1735696210.378044]: received mask_obs with header seq: 499 +stamp: + secs: 1735696210 + nsecs: 379311800 +frame_id: '' and grasped_object_mask header seq: 105399 +stamp: + secs: 1735696210 + nsecs: 152149677 +frame_id: "camera_color_optical_frame" +[INFO] [1735696210.576427]: received mask_obs with header seq: 500 +stamp: + secs: 1735696210 + nsecs: 578047990 +frame_id: '' and grasped_object_mask header seq: 105405 +stamp: + secs: 1735696210 + nsecs: 351990461 +frame_id: "camera_color_optical_frame" +[INFO] [1735696210.776214]: received mask_obs with header seq: 501 +stamp: + secs: 1735696210 + nsecs: 777418613 +frame_id: '' and grasped_object_mask header seq: 105411 +stamp: + secs: 1735696210 + nsecs: 551832914 +frame_id: "camera_color_optical_frame" +[INFO] [1735696210.979002]: received mask_obs with header seq: 502 +stamp: + secs: 1735696210 + nsecs: 980411291 +frame_id: '' and grasped_object_mask header seq: 105417 +stamp: + secs: 1735696210 + nsecs: 751675606 +frame_id: "camera_color_optical_frame" +[INFO] [1735696211.175341]: received mask_obs with header seq: 503 +stamp: + secs: 1735696211 + nsecs: 176628589 +frame_id: '' and grasped_object_mask header seq: 105423 +stamp: + secs: 1735696210 + nsecs: 951519251 +frame_id: "camera_color_optical_frame" +[INFO] [1735696211.374947]: received mask_obs with header seq: 504 +stamp: + secs: 1735696211 + nsecs: 376524209 +frame_id: '' and grasped_object_mask header seq: 105429 +stamp: + secs: 1735696211 + nsecs: 151362419 +frame_id: "camera_color_optical_frame" +[INFO] [1735696211.577768]: received mask_obs with header seq: 505 +stamp: + secs: 1735696211 + nsecs: 579355716 +frame_id: '' and grasped_object_mask header seq: 105435 +stamp: + secs: 1735696211 + nsecs: 351204634 +frame_id: "camera_color_optical_frame" +[INFO] [1735696211.781318]: received mask_obs with header seq: 506 +stamp: + secs: 1735696211 + nsecs: 782621383 +frame_id: '' and grasped_object_mask header seq: 105441 +stamp: + secs: 1735696211 + nsecs: 551043034 +frame_id: "camera_color_optical_frame" +[INFO] [1735696211.978264]: received mask_obs with header seq: 507 +stamp: + secs: 1735696211 + nsecs: 979573011 +frame_id: '' and grasped_object_mask header seq: 105447 +stamp: + secs: 1735696211 + nsecs: 750881195 +frame_id: "camera_color_optical_frame" +[INFO] [1735696212.176817]: received mask_obs with header seq: 508 +stamp: + secs: 1735696212 + nsecs: 178441524 +frame_id: '' and grasped_object_mask header seq: 105453 +stamp: + secs: 1735696211 + nsecs: 950719595 +frame_id: "camera_color_optical_frame" +[INFO] [1735696212.375267]: received mask_obs with header seq: 509 +stamp: + secs: 1735696212 + nsecs: 375485420 +frame_id: '' and grasped_object_mask header seq: 105459 +stamp: + secs: 1735696212 + nsecs: 150558949 +frame_id: "camera_color_optical_frame" +[INFO] [1735696212.576185]: received mask_obs with header seq: 510 +stamp: + secs: 1735696212 + nsecs: 577283143 +frame_id: '' and grasped_object_mask header seq: 105465 +stamp: + secs: 1735696212 + nsecs: 350397110 +frame_id: "camera_color_optical_frame" +[INFO] [1735696212.777636]: received mask_obs with header seq: 511 +stamp: + secs: 1735696212 + nsecs: 779152154 +frame_id: '' and grasped_object_mask header seq: 105471 +stamp: + secs: 1735696212 + nsecs: 550237179 +frame_id: "camera_color_optical_frame" +[INFO] [1735696212.976706]: received mask_obs with header seq: 512 +stamp: + secs: 1735696212 + nsecs: 977333068 +frame_id: '' and grasped_object_mask header seq: 105477 +stamp: + secs: 1735696212 + nsecs: 750076771 +frame_id: "camera_color_optical_frame" +[INFO] [1735696213.174280]: received mask_obs with header seq: 513 +stamp: + secs: 1735696213 + nsecs: 174793004 +frame_id: '' and grasped_object_mask header seq: 105483 +stamp: + secs: 1735696212 + nsecs: 949916601 +frame_id: "camera_color_optical_frame" +[INFO] [1735696213.376681]: received mask_obs with header seq: 514 +stamp: + secs: 1735696213 + nsecs: 378219842 +frame_id: '' and grasped_object_mask header seq: 105489 +stamp: + secs: 1735696213 + nsecs: 149756670 +frame_id: "camera_color_optical_frame" +[INFO] [1735696213.575231]: received mask_obs with header seq: 515 +stamp: + secs: 1735696213 + nsecs: 576232910 +frame_id: '' and grasped_object_mask header seq: 105495 +stamp: + secs: 1735696213 + nsecs: 349596977 +frame_id: "camera_color_optical_frame" +[INFO] [1735696213.775109]: received mask_obs with header seq: 516 +stamp: + secs: 1735696213 + nsecs: 775351047 +frame_id: '' and grasped_object_mask header seq: 105501 +stamp: + secs: 1735696213 + nsecs: 549431801 +frame_id: "camera_color_optical_frame" +[INFO] [1735696213.976872]: received mask_obs with header seq: 517 +stamp: + secs: 1735696213 + nsecs: 978525876 +frame_id: '' and grasped_object_mask header seq: 105507 +stamp: + secs: 1735696213 + nsecs: 749266863 +frame_id: "camera_color_optical_frame" +[INFO] [1735696214.175362]: received mask_obs with header seq: 518 +stamp: + secs: 1735696214 + nsecs: 175211668 +frame_id: '' and grasped_object_mask header seq: 105513 +stamp: + secs: 1735696213 + nsecs: 949101210 +frame_id: "camera_color_optical_frame" +[INFO] [1735696214.374358]: received mask_obs with header seq: 519 +stamp: + secs: 1735696214 + nsecs: 374533414 +frame_id: '' and grasped_object_mask header seq: 105519 +stamp: + secs: 1735696214 + nsecs: 148935795 +frame_id: "camera_color_optical_frame" +[INFO] [1735696214.576096]: received mask_obs with header seq: 520 +stamp: + secs: 1735696214 + nsecs: 577553510 +frame_id: '' and grasped_object_mask header seq: 105525 +stamp: + secs: 1735696214 + nsecs: 348770618 +frame_id: "camera_color_optical_frame" +[INFO] [1735696214.772044]: received mask_obs with header seq: 521 +stamp: + secs: 1735696214 + nsecs: 773697614 +frame_id: '' and grasped_object_mask header seq: 105531 +stamp: + secs: 1735696214 + nsecs: 548608303 +frame_id: "camera_color_optical_frame" +[INFO] [1735696214.974841]: received mask_obs with header seq: 522 +stamp: + secs: 1735696214 + nsecs: 974403142 +frame_id: '' and grasped_object_mask header seq: 105537 +stamp: + secs: 1735696214 + nsecs: 748447418 +frame_id: "camera_color_optical_frame" +[INFO] [1735696215.174884]: received mask_obs with header seq: 523 +stamp: + secs: 1735696215 + nsecs: 175703048 +frame_id: '' and grasped_object_mask header seq: 105543 +stamp: + secs: 1735696214 + nsecs: 948285580 +frame_id: "camera_color_optical_frame" +[INFO] [1735696215.374505]: received mask_obs with header seq: 524 +stamp: + secs: 1735696215 + nsecs: 375487089 +frame_id: '' and grasped_object_mask header seq: 105549 +stamp: + secs: 1735696215 + nsecs: 148123741 +frame_id: "camera_color_optical_frame" +[INFO] [1735696215.574090]: received mask_obs with header seq: 525 +stamp: + secs: 1735696215 + nsecs: 575752735 +frame_id: '' and grasped_object_mask header seq: 105555 +stamp: + secs: 1735696215 + nsecs: 347962141 +frame_id: "camera_color_optical_frame" +[INFO] [1735696215.774159]: received mask_obs with header seq: 526 +stamp: + secs: 1735696215 + nsecs: 774940252 +frame_id: '' and grasped_object_mask header seq: 105561 +stamp: + secs: 1735696215 + nsecs: 547801971 +frame_id: "camera_color_optical_frame" +[INFO] [1735696215.973637]: received mask_obs with header seq: 527 +stamp: + secs: 1735696215 + nsecs: 974933862 +frame_id: '' and grasped_object_mask header seq: 105567 +stamp: + secs: 1735696215 + nsecs: 747642040 +frame_id: "camera_color_optical_frame" +[INFO] [1735696216.182748]: received mask_obs with header seq: 528 +stamp: + secs: 1735696216 + nsecs: 184357404 +frame_id: '' and grasped_object_mask header seq: 105573 +stamp: + secs: 1735696215 + nsecs: 947483063 +frame_id: "camera_color_optical_frame" +[INFO] [1735696216.380522]: received mask_obs with header seq: 529 +stamp: + secs: 1735696216 + nsecs: 382119417 +frame_id: '' and grasped_object_mask header seq: 105579 +stamp: + secs: 1735696216 + nsecs: 147323370 +frame_id: "camera_color_optical_frame" +[INFO] [1735696216.577380]: received mask_obs with header seq: 530 +stamp: + secs: 1735696216 + nsecs: 579007625 +frame_id: '' and grasped_object_mask header seq: 105585 +stamp: + secs: 1735696216 + nsecs: 347163677 +frame_id: "camera_color_optical_frame" +[INFO] [1735696216.772599]: received mask_obs with header seq: 531 +stamp: + secs: 1735696216 + nsecs: 774257183 +frame_id: '' and grasped_object_mask header seq: 105591 +stamp: + secs: 1735696216 + nsecs: 547000885 +frame_id: "camera_color_optical_frame" +[INFO] [1735696216.816677]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735696216.975821]: received mask_obs with header seq: 532 +stamp: + secs: 1735696216 + nsecs: 975817918 +frame_id: '' and grasped_object_mask header seq: 105597 +stamp: + secs: 1735696216 + nsecs: 746838331 +frame_id: "camera_color_optical_frame" +[INFO] [1735696217.671890]: Stored episode 4. +[INFO] [1735696217.677274]: Saving video... +[INFO] [1735696217.902881]: Video saved! +Starting episode 4 +[INFO] [1735696217.953306]: resetting environment +[INFO] [1735696217.974621]: cleared current plan +[INFO] [1735696217.976767]: moving to home +[INFO] [1735696221.584185]: reached home +[INFO] [1735696221.594269]: reset action history +[INFO] [1735696224.456073]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735696235.018865]: resetting environment +[INFO] [1735696235.050853]: cleared current plan +[INFO] [1735696235.053344]: moving to home +[INFO] [1735696236.061364]: reached home +[INFO] [1735696236.071807]: reset action history +[INFO] [1735696238.859940]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/episode_rosbags/episode_4_2024-12-31-20-50-38.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1737022.log +started roslaunch server http://158.130.50.37:44003/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1738126] +process[print_text-14]: started with pid [1738128] +process[rosbag_record-15]: started with pid [1738136] +started bagging! +[INFO] [1735696241.771117]: received mask_obs with header seq: 533 +stamp: + secs: 1735696241 + nsecs: 772533416 +frame_id: '' and grasped_object_mask header seq: 106342 +stamp: + secs: 1735696241 + nsecs: 559982061 +frame_id: "camera_color_optical_frame" +[INFO] [1735696241.965873]: received mask_obs with header seq: 534 +stamp: + secs: 1735696241 + nsecs: 967484235 +frame_id: '' and grasped_object_mask header seq: 106348 +stamp: + secs: 1735696241 + nsecs: 759817362 +frame_id: "camera_color_optical_frame" +[INFO] [1735696242.168968]: received mask_obs with header seq: 535 +stamp: + secs: 1735696242 + nsecs: 170681238 +frame_id: '' and grasped_object_mask header seq: 106354 +stamp: + secs: 1735696241 + nsecs: 959652185 +frame_id: "camera_color_optical_frame" +[INFO] [1735696242.368796]: received mask_obs with header seq: 536 +stamp: + secs: 1735696242 + nsecs: 370512247 +frame_id: '' and grasped_object_mask header seq: 106360 +stamp: + secs: 1735696242 + nsecs: 159487009 +frame_id: "camera_color_optical_frame" +[INFO] [1735696242.577041]: received mask_obs with header seq: 537 +stamp: + secs: 1735696242 + nsecs: 577661991 +frame_id: '' and grasped_object_mask header seq: 106366 +stamp: + secs: 1735696242 + nsecs: 359322548 +frame_id: "camera_color_optical_frame" +[INFO] [1735696242.781641]: received mask_obs with header seq: 538 +stamp: + secs: 1735696242 + nsecs: 782592535 +frame_id: '' and grasped_object_mask header seq: 106372 +stamp: + secs: 1735696242 + nsecs: 559161901 +frame_id: "camera_color_optical_frame" +[INFO] [1735696242.991638]: received mask_obs with header seq: 539 +stamp: + secs: 1735696242 + nsecs: 992936134 +frame_id: '' and grasped_object_mask header seq: 106378 +stamp: + secs: 1735696242 + nsecs: 759001493 +frame_id: "camera_color_optical_frame" +[INFO] [1735696243.193515]: received mask_obs with header seq: 540 +stamp: + secs: 1735696243 + nsecs: 194890737 +frame_id: '' and grasped_object_mask header seq: 106384 +stamp: + secs: 1735696242 + nsecs: 958841085 +frame_id: "camera_color_optical_frame" +[INFO] [1735696243.389544]: received mask_obs with header seq: 541 +stamp: + secs: 1735696243 + nsecs: 391283750 +frame_id: '' and grasped_object_mask header seq: 106390 +stamp: + secs: 1735696243 + nsecs: 158680677 +frame_id: "camera_color_optical_frame" +[INFO] [1735696243.589571]: received mask_obs with header seq: 542 +stamp: + secs: 1735696243 + nsecs: 591217994 +frame_id: '' and grasped_object_mask header seq: 106396 +stamp: + secs: 1735696243 + nsecs: 358520269 +frame_id: "camera_color_optical_frame" +[INFO] [1735696243.791563]: received mask_obs with header seq: 543 +stamp: + secs: 1735696243 + nsecs: 793220758 +frame_id: '' and grasped_object_mask header seq: 106402 +stamp: + secs: 1735696243 + nsecs: 558369160 +frame_id: "camera_color_optical_frame" +[INFO] [1735696243.987825]: received mask_obs with header seq: 544 +stamp: + secs: 1735696243 + nsecs: 989472150 +frame_id: '' and grasped_object_mask header seq: 106408 +stamp: + secs: 1735696243 + nsecs: 758217573 +frame_id: "camera_color_optical_frame" +[INFO] [1735696244.186148]: received mask_obs with header seq: 545 +stamp: + secs: 1735696244 + nsecs: 187653303 +frame_id: '' and grasped_object_mask header seq: 106414 +stamp: + secs: 1735696243 + nsecs: 958066225 +frame_id: "camera_color_optical_frame" +[INFO] [1735696244.388876]: received mask_obs with header seq: 546 +stamp: + secs: 1735696244 + nsecs: 390536546 +frame_id: '' and grasped_object_mask header seq: 106420 +stamp: + secs: 1735696244 + nsecs: 157915354 +frame_id: "camera_color_optical_frame" +[INFO] [1735696244.591084]: received mask_obs with header seq: 547 +stamp: + secs: 1735696244 + nsecs: 592797756 +frame_id: '' and grasped_object_mask header seq: 106426 +stamp: + secs: 1735696244 + nsecs: 357764721 +frame_id: "camera_color_optical_frame" +[INFO] [1735696244.788289]: received mask_obs with header seq: 548 +stamp: + secs: 1735696244 + nsecs: 789764881 +frame_id: '' and grasped_object_mask header seq: 106432 +stamp: + secs: 1735696244 + nsecs: 559038162 +frame_id: "camera_color_optical_frame" +[INFO] [1735696244.987991]: received mask_obs with header seq: 549 +stamp: + secs: 1735696244 + nsecs: 989689588 +frame_id: '' and grasped_object_mask header seq: 106438 +stamp: + secs: 1735696244 + nsecs: 761218071 +frame_id: "camera_color_optical_frame" +[INFO] [1735696245.190322]: received mask_obs with header seq: 550 +stamp: + secs: 1735696245 + nsecs: 191806554 +frame_id: '' and grasped_object_mask header seq: 106444 +stamp: + secs: 1735696244 + nsecs: 963485241 +frame_id: "camera_color_optical_frame" +[INFO] [1735696245.388514]: received mask_obs with header seq: 551 +stamp: + secs: 1735696245 + nsecs: 390029430 +frame_id: '' and grasped_object_mask header seq: 106450 +stamp: + secs: 1735696245 + nsecs: 165838480 +frame_id: "camera_color_optical_frame" +[INFO] [1735696245.589809]: received mask_obs with header seq: 552 +stamp: + secs: 1735696245 + nsecs: 591501712 +frame_id: '' and grasped_object_mask header seq: 106456 +stamp: + secs: 1735696245 + nsecs: 368277788 +frame_id: "camera_color_optical_frame" +[INFO] [1735696245.788139]: received mask_obs with header seq: 553 +stamp: + secs: 1735696245 + nsecs: 789862155 +frame_id: '' and grasped_object_mask header seq: 106462 +stamp: + secs: 1735696245 + nsecs: 568972111 +frame_id: "camera_color_optical_frame" +[INFO] [1735696245.988316]: received mask_obs with header seq: 554 +stamp: + secs: 1735696245 + nsecs: 989977598 +frame_id: '' and grasped_object_mask header seq: 106468 +stamp: + secs: 1735696245 + nsecs: 768592596 +frame_id: "camera_color_optical_frame" +[INFO] [1735696246.198012]: received mask_obs with header seq: 555 +stamp: + secs: 1735696246 + nsecs: 199692487 +frame_id: '' and grasped_object_mask header seq: 106474 +stamp: + secs: 1735696245 + nsecs: 968201637 +frame_id: "camera_color_optical_frame" +[INFO] [1735696246.396828]: received mask_obs with header seq: 556 +stamp: + secs: 1735696246 + nsecs: 398350954 +frame_id: '' and grasped_object_mask header seq: 106480 +stamp: + secs: 1735696246 + nsecs: 167800426 +frame_id: "camera_color_optical_frame" +[INFO] [1735696246.602156]: received mask_obs with header seq: 557 +stamp: + secs: 1735696246 + nsecs: 603899717 +frame_id: '' and grasped_object_mask header seq: 106486 +stamp: + secs: 1735696246 + nsecs: 367387295 +frame_id: "camera_color_optical_frame" +[INFO] [1735696246.798598]: received mask_obs with header seq: 558 +stamp: + secs: 1735696246 + nsecs: 800302743 +frame_id: '' and grasped_object_mask header seq: 106492 +stamp: + secs: 1735696246 + nsecs: 566992521 +frame_id: "camera_color_optical_frame" +[INFO] [1735696246.996622]: received mask_obs with header seq: 559 +stamp: + secs: 1735696246 + nsecs: 998310565 +frame_id: '' and grasped_object_mask header seq: 106498 +stamp: + secs: 1735696246 + nsecs: 766603470 +frame_id: "camera_color_optical_frame" +[INFO] [1735696247.189584]: received mask_obs with header seq: 560 +stamp: + secs: 1735696247 + nsecs: 191312074 +frame_id: '' and grasped_object_mask header seq: 106504 +stamp: + secs: 1735696246 + nsecs: 966203451 +frame_id: "camera_color_optical_frame" +[INFO] [1735696247.389028]: received mask_obs with header seq: 561 +stamp: + secs: 1735696247 + nsecs: 390770673 +frame_id: '' and grasped_object_mask header seq: 106510 +stamp: + secs: 1735696247 + nsecs: 165791988 +frame_id: "camera_color_optical_frame" +[INFO] [1735696247.589459]: received mask_obs with header seq: 562 +stamp: + secs: 1735696247 + nsecs: 591034412 +frame_id: '' and grasped_object_mask header seq: 106516 +stamp: + secs: 1735696247 + nsecs: 365369797 +frame_id: "camera_color_optical_frame" +[INFO] [1735696247.788505]: received mask_obs with header seq: 563 +stamp: + secs: 1735696247 + nsecs: 790178537 +frame_id: '' and grasped_object_mask header seq: 106522 +stamp: + secs: 1735696247 + nsecs: 564973116 +frame_id: "camera_color_optical_frame" +[INFO] [1735696247.989634]: received mask_obs with header seq: 564 +stamp: + secs: 1735696247 + nsecs: 991345167 +frame_id: '' and grasped_object_mask header seq: 106528 +stamp: + secs: 1735696247 + nsecs: 764587641 +frame_id: "camera_color_optical_frame" +[INFO] [1735696248.188764]: received mask_obs with header seq: 565 +stamp: + secs: 1735696248 + nsecs: 190312623 +frame_id: '' and grasped_object_mask header seq: 106534 +stamp: + secs: 1735696247 + nsecs: 964190722 +frame_id: "camera_color_optical_frame" +[INFO] [1735696248.387372]: received mask_obs with header seq: 566 +stamp: + secs: 1735696248 + nsecs: 389020442 +frame_id: '' and grasped_object_mask header seq: 106540 +stamp: + secs: 1735696248 + nsecs: 163782120 +frame_id: "camera_color_optical_frame" +[INFO] [1735696248.587993]: received mask_obs with header seq: 567 +stamp: + secs: 1735696248 + nsecs: 589682102 +frame_id: '' and grasped_object_mask header seq: 106546 +stamp: + secs: 1735696248 + nsecs: 363362551 +frame_id: "camera_color_optical_frame" +[INFO] [1735696248.788298]: received mask_obs with header seq: 568 +stamp: + secs: 1735696248 + nsecs: 790008544 +frame_id: '' and grasped_object_mask header seq: 106552 +stamp: + secs: 1735696248 + nsecs: 562966108 +frame_id: "camera_color_optical_frame" +[INFO] [1735696248.987416]: received mask_obs with header seq: 569 +stamp: + secs: 1735696248 + nsecs: 989084959 +frame_id: '' and grasped_object_mask header seq: 106558 +stamp: + secs: 1735696248 + nsecs: 762577772 +frame_id: "camera_color_optical_frame" +[INFO] [1735696249.186681]: received mask_obs with header seq: 570 +stamp: + secs: 1735696249 + nsecs: 188303947 +frame_id: '' and grasped_object_mask header seq: 106564 +stamp: + secs: 1735696248 + nsecs: 962177753 +frame_id: "camera_color_optical_frame" +[INFO] [1735696249.389338]: received mask_obs with header seq: 571 +stamp: + secs: 1735696249 + nsecs: 390992403 +frame_id: '' and grasped_object_mask header seq: 106570 +stamp: + secs: 1735696249 + nsecs: 161766291 +frame_id: "camera_color_optical_frame" +[INFO] [1735696249.589149]: received mask_obs with header seq: 572 +stamp: + secs: 1735696249 + nsecs: 590595245 +frame_id: '' and grasped_object_mask header seq: 106576 +stamp: + secs: 1735696249 + nsecs: 361343622 +frame_id: "camera_color_optical_frame" +[INFO] [1735696249.788441]: received mask_obs with header seq: 573 +stamp: + secs: 1735696249 + nsecs: 790089845 +frame_id: '' and grasped_object_mask header seq: 106582 +stamp: + secs: 1735696249 + nsecs: 560944796 +frame_id: "camera_color_optical_frame" +[INFO] [1735696249.989569]: received mask_obs with header seq: 574 +stamp: + secs: 1735696249 + nsecs: 991280317 +frame_id: '' and grasped_object_mask header seq: 106588 +stamp: + secs: 1735696249 + nsecs: 760558128 +frame_id: "camera_color_optical_frame" +[INFO] [1735696250.186262]: received mask_obs with header seq: 575 +stamp: + secs: 1735696250 + nsecs: 187862157 +frame_id: '' and grasped_object_mask header seq: 106594 +stamp: + secs: 1735696249 + nsecs: 960158587 +frame_id: "camera_color_optical_frame" +[INFO] [1735696250.397817]: received mask_obs with header seq: 576 +stamp: + secs: 1735696250 + nsecs: 399507761 +frame_id: '' and grasped_object_mask header seq: 106600 +stamp: + secs: 1735696250 + nsecs: 159747839 +frame_id: "camera_color_optical_frame" +[INFO] [1735696250.594947]: received mask_obs with header seq: 577 +stamp: + secs: 1735696250 + nsecs: 596680402 +frame_id: '' and grasped_object_mask header seq: 106606 +stamp: + secs: 1735696250 + nsecs: 359325647 +frame_id: "camera_color_optical_frame" +[INFO] [1735696250.789877]: received mask_obs with header seq: 578 +stamp: + secs: 1735696250 + nsecs: 791322946 +frame_id: '' and grasped_object_mask header seq: 106612 +stamp: + secs: 1735696250 + nsecs: 558928490 +frame_id: "camera_color_optical_frame" +[INFO] [1735696250.988576]: received mask_obs with header seq: 579 +stamp: + secs: 1735696250 + nsecs: 990322589 +frame_id: '' and grasped_object_mask header seq: 106618 +stamp: + secs: 1735696250 + nsecs: 758542538 +frame_id: "camera_color_optical_frame" +[INFO] [1735696251.188612]: received mask_obs with header seq: 580 +stamp: + secs: 1735696251 + nsecs: 190233707 +frame_id: '' and grasped_object_mask header seq: 106624 +stamp: + secs: 1735696250 + nsecs: 958146334 +frame_id: "camera_color_optical_frame" +[INFO] [1735696251.388898]: received mask_obs with header seq: 581 +stamp: + secs: 1735696251 + nsecs: 390430688 +frame_id: '' and grasped_object_mask header seq: 106630 +stamp: + secs: 1735696251 + nsecs: 157737494 +frame_id: "camera_color_optical_frame" +[INFO] [1735696251.588670]: received mask_obs with header seq: 582 +stamp: + secs: 1735696251 + nsecs: 590292215 +frame_id: '' and grasped_object_mask header seq: 106636 +stamp: + secs: 1735696251 + nsecs: 357317209 +frame_id: "camera_color_optical_frame" +[INFO] [1735696251.788538]: received mask_obs with header seq: 583 +stamp: + secs: 1735696251 + nsecs: 790091753 +frame_id: '' and grasped_object_mask header seq: 106642 +stamp: + secs: 1735696251 + nsecs: 556917429 +frame_id: "camera_color_optical_frame" +[INFO] [1735696251.988139]: received mask_obs with header seq: 584 +stamp: + secs: 1735696251 + nsecs: 989873886 +frame_id: '' and grasped_object_mask header seq: 106648 +stamp: + secs: 1735696251 + nsecs: 756526470 +frame_id: "camera_color_optical_frame" +[INFO] [1735696252.187300]: received mask_obs with header seq: 585 +stamp: + secs: 1735696252 + nsecs: 188962697 +frame_id: '' and grasped_object_mask header seq: 106654 +stamp: + secs: 1735696251 + nsecs: 956123829 +frame_id: "camera_color_optical_frame" +[INFO] [1735696252.387691]: received mask_obs with header seq: 586 +stamp: + secs: 1735696252 + nsecs: 389434099 +frame_id: '' and grasped_object_mask header seq: 106660 +stamp: + secs: 1735696252 + nsecs: 155709267 +frame_id: "camera_color_optical_frame" +[INFO] [1735696252.588809]: received mask_obs with header seq: 587 +stamp: + secs: 1735696252 + nsecs: 589878082 +frame_id: '' and grasped_object_mask header seq: 106666 +stamp: + secs: 1735696252 + nsecs: 355284452 +frame_id: "camera_color_optical_frame" +[INFO] [1735696252.789418]: received mask_obs with header seq: 588 +stamp: + secs: 1735696252 + nsecs: 790990591 +frame_id: '' and grasped_object_mask header seq: 106672 +stamp: + secs: 1735696252 + nsecs: 554882765 +frame_id: "camera_color_optical_frame" +[INFO] [1735696252.988508]: received mask_obs with header seq: 589 +stamp: + secs: 1735696252 + nsecs: 990000486 +frame_id: '' and grasped_object_mask header seq: 106678 +stamp: + secs: 1735696252 + nsecs: 754492760 +frame_id: "camera_color_optical_frame" +[INFO] [1735696253.187130]: received mask_obs with header seq: 590 +stamp: + secs: 1735696253 + nsecs: 188674449 +frame_id: '' and grasped_object_mask header seq: 106684 +stamp: + secs: 1735696252 + nsecs: 954090834 +frame_id: "camera_color_optical_frame" +[INFO] [1735696253.386495]: received mask_obs with header seq: 591 +stamp: + secs: 1735696253 + nsecs: 388021945 +frame_id: '' and grasped_object_mask header seq: 106690 +stamp: + secs: 1735696253 + nsecs: 153677464 +frame_id: "camera_color_optical_frame" +[INFO] [1735696253.589028]: received mask_obs with header seq: 592 +stamp: + secs: 1735696253 + nsecs: 590715885 +frame_id: '' and grasped_object_mask header seq: 106696 +stamp: + secs: 1735696253 + nsecs: 353252411 +frame_id: "camera_color_optical_frame" +[INFO] [1735696253.784936]: received mask_obs with header seq: 593 +stamp: + secs: 1735696253 + nsecs: 786737680 +frame_id: '' and grasped_object_mask header seq: 106702 +stamp: + secs: 1735696253 + nsecs: 552851200 +frame_id: "camera_color_optical_frame" +[INFO] [1735696253.986545]: received mask_obs with header seq: 594 +stamp: + secs: 1735696253 + nsecs: 988323688 +frame_id: '' and grasped_object_mask header seq: 106708 +stamp: + secs: 1735696253 + nsecs: 752459764 +frame_id: "camera_color_optical_frame" +[INFO] [1735696254.188032]: received mask_obs with header seq: 595 +stamp: + secs: 1735696254 + nsecs: 189769983 +frame_id: '' and grasped_object_mask header seq: 106714 +stamp: + secs: 1735696253 + nsecs: 952056408 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 101765800 for nearest: 1735696254253407169 - target: 1735696254151641369 at idx 207 +[INFO] [1735696254.387117]: received mask_obs with header seq: 596 +stamp: + secs: 1735696254 + nsecs: 388692378 +frame_id: '' and grasped_object_mask header seq: 106720 +stamp: + secs: 1735696254 + nsecs: 151641369 +frame_id: "camera_color_optical_frame" +[INFO] [1735696254.586755]: received mask_obs with header seq: 597 +stamp: + secs: 1735696254 + nsecs: 588116884 +frame_id: '' and grasped_object_mask header seq: 106726 +stamp: + secs: 1735696254 + nsecs: 351214647 +frame_id: "camera_color_optical_frame" +[INFO] [1735696254.787405]: received mask_obs with header seq: 598 +stamp: + secs: 1735696254 + nsecs: 789183139 +frame_id: '' and grasped_object_mask header seq: 106732 +stamp: + secs: 1735696254 + nsecs: 550810814 +frame_id: "camera_color_optical_frame" +[INFO] [1735696254.997221]: received mask_obs with header seq: 599 +stamp: + secs: 1735696254 + nsecs: 998825788 +frame_id: '' and grasped_object_mask header seq: 106738 +stamp: + secs: 1735696254 + nsecs: 750418663 +frame_id: "camera_color_optical_frame" +[INFO] [1735696255.196528]: received mask_obs with header seq: 600 +stamp: + secs: 1735696255 + nsecs: 198300123 +frame_id: '' and grasped_object_mask header seq: 106744 +stamp: + secs: 1735696254 + nsecs: 950013876 +frame_id: "camera_color_optical_frame" +[INFO] [1735696255.391808]: received mask_obs with header seq: 601 +stamp: + secs: 1735696255 + nsecs: 393061637 +frame_id: '' and grasped_object_mask header seq: 106750 +stamp: + secs: 1735696255 + nsecs: 149597406 +frame_id: "camera_color_optical_frame" +[INFO] [1735696255.588496]: received mask_obs with header seq: 602 +stamp: + secs: 1735696255 + nsecs: 590137958 +frame_id: '' and grasped_object_mask header seq: 106756 +stamp: + secs: 1735696255 + nsecs: 349169254 +frame_id: "camera_color_optical_frame" +[INFO] [1735696255.787543]: received mask_obs with header seq: 603 +stamp: + secs: 1735696255 + nsecs: 789203405 +frame_id: '' and grasped_object_mask header seq: 106762 +stamp: + secs: 1735696255 + nsecs: 548764467 +frame_id: "camera_color_optical_frame" +[INFO] [1735696255.987441]: received mask_obs with header seq: 604 +stamp: + secs: 1735696255 + nsecs: 989147424 +frame_id: '' and grasped_object_mask header seq: 106768 +stamp: + secs: 1735696255 + nsecs: 748371601 +frame_id: "camera_color_optical_frame" +[INFO] [1735696256.190196]: received mask_obs with header seq: 605 +stamp: + secs: 1735696256 + nsecs: 191397190 +frame_id: '' and grasped_object_mask header seq: 106774 +stamp: + secs: 1735696255 + nsecs: 947967768 +frame_id: "camera_color_optical_frame" +[INFO] [1735696256.387609]: received mask_obs with header seq: 606 +stamp: + secs: 1735696256 + nsecs: 389339447 +frame_id: '' and grasped_object_mask header seq: 106780 +stamp: + secs: 1735696256 + nsecs: 147551298 +frame_id: "camera_color_optical_frame" +[INFO] [1735696256.589238]: received mask_obs with header seq: 607 +stamp: + secs: 1735696256 + nsecs: 590592622 +frame_id: '' and grasped_object_mask header seq: 106786 +stamp: + secs: 1735696256 + nsecs: 347122908 +frame_id: "camera_color_optical_frame" +[INFO] [1735696256.788214]: received mask_obs with header seq: 608 +stamp: + secs: 1735696256 + nsecs: 789729118 +frame_id: '' and grasped_object_mask header seq: 106792 +stamp: + secs: 1735696256 + nsecs: 546718597 +frame_id: "camera_color_optical_frame" +[INFO] [1735696256.986797]: received mask_obs with header seq: 609 +stamp: + secs: 1735696256 + nsecs: 988602161 +frame_id: '' and grasped_object_mask header seq: 106798 +stamp: + secs: 1735696256 + nsecs: 746326447 +frame_id: "camera_color_optical_frame" +[INFO] [1735696257.189135]: received mask_obs with header seq: 610 +stamp: + secs: 1735696257 + nsecs: 190808057 +frame_id: '' and grasped_object_mask header seq: 106804 +stamp: + secs: 1735696256 + nsecs: 945922375 +frame_id: "camera_color_optical_frame" +[INFO] [1735696257.395520]: received mask_obs with header seq: 611 +stamp: + secs: 1735696257 + nsecs: 397264242 +frame_id: '' and grasped_object_mask header seq: 106810 +stamp: + secs: 1735696257 + nsecs: 145507574 +frame_id: "camera_color_optical_frame" +[INFO] [1735696257.590673]: received mask_obs with header seq: 612 +stamp: + secs: 1735696257 + nsecs: 592409610 +frame_id: '' and grasped_object_mask header seq: 106816 +stamp: + secs: 1735696257 + nsecs: 345080137 +frame_id: "camera_color_optical_frame" +[INFO] [1735696257.788546]: received mask_obs with header seq: 613 +stamp: + secs: 1735696257 + nsecs: 790180206 +frame_id: '' and grasped_object_mask header seq: 106822 +stamp: + secs: 1735696257 + nsecs: 544673204 +frame_id: "camera_color_optical_frame" +[INFO] [1735696257.986153]: received mask_obs with header seq: 614 +stamp: + secs: 1735696257 + nsecs: 987727642 +frame_id: '' and grasped_object_mask header seq: 106828 +stamp: + secs: 1735696257 + nsecs: 744275570 +frame_id: "camera_color_optical_frame" +[INFO] [1735696258.196631]: received mask_obs with header seq: 615 +stamp: + secs: 1735696258 + nsecs: 198391199 +frame_id: '' and grasped_object_mask header seq: 106834 +stamp: + secs: 1735696257 + nsecs: 943866253 +frame_id: "camera_color_optical_frame" +[INFO] [1735696258.394090]: received mask_obs with header seq: 616 +stamp: + secs: 1735696258 + nsecs: 395806789 +frame_id: '' and grasped_object_mask header seq: 106840 +stamp: + secs: 1735696258 + nsecs: 143444538 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 110398617 for nearest: 1735696258453410473 - target: 1735696258343011856 at idx 249 +[INFO] [1735696258.588952]: received mask_obs with header seq: 617 +stamp: + secs: 1735696258 + nsecs: 590680122 +frame_id: '' and grasped_object_mask header seq: 106846 +stamp: + secs: 1735696258 + nsecs: 343011856 +frame_id: "camera_color_optical_frame" +[INFO] [1735696258.795953]: received mask_obs with header seq: 618 +stamp: + secs: 1735696258 + nsecs: 797719955 +frame_id: '' and grasped_object_mask header seq: 106852 +stamp: + secs: 1735696258 + nsecs: 542604446 +frame_id: "camera_color_optical_frame" +[INFO] [1735696258.992350]: received mask_obs with header seq: 619 +stamp: + secs: 1735696258 + nsecs: 994120359 +frame_id: '' and grasped_object_mask header seq: 106858 +stamp: + secs: 1735696258 + nsecs: 742210865 +frame_id: "camera_color_optical_frame" +[INFO] [1735696259.198863]: received mask_obs with header seq: 620 +stamp: + secs: 1735696259 + nsecs: 200588941 +frame_id: '' and grasped_object_mask header seq: 106864 +stamp: + secs: 1735696258 + nsecs: 941805601 +frame_id: "camera_color_optical_frame" +[INFO] [1735696259.396640]: received mask_obs with header seq: 621 +stamp: + secs: 1735696259 + nsecs: 398153781 +frame_id: '' and grasped_object_mask header seq: 106870 +stamp: + secs: 1735696259 + nsecs: 141388655 +frame_id: "camera_color_optical_frame" +[INFO] [1735696259.593181]: received mask_obs with header seq: 622 +stamp: + secs: 1735696259 + nsecs: 594929218 +frame_id: '' and grasped_object_mask header seq: 106876 +stamp: + secs: 1735696259 + nsecs: 340959549 +frame_id: "camera_color_optical_frame" +[INFO] [1735696259.787094]: received mask_obs with header seq: 623 +stamp: + secs: 1735696259 + nsecs: 788730621 +frame_id: '' and grasped_object_mask header seq: 106882 +stamp: + secs: 1735696259 + nsecs: 540556192 +frame_id: "camera_color_optical_frame" +[INFO] [1735696259.987023]: received mask_obs with header seq: 624 +stamp: + secs: 1735696259 + nsecs: 988471984 +frame_id: '' and grasped_object_mask header seq: 106888 +stamp: + secs: 1735696259 + nsecs: 740167856 +frame_id: "camera_color_optical_frame" +[INFO] [1735696260.187441]: received mask_obs with header seq: 625 +stamp: + secs: 1735696260 + nsecs: 189112901 +frame_id: '' and grasped_object_mask header seq: 106894 +stamp: + secs: 1735696259 + nsecs: 939767361 +frame_id: "camera_color_optical_frame" +[INFO] [1735696260.386248]: received mask_obs with header seq: 626 +stamp: + secs: 1735696260 + nsecs: 388013362 +frame_id: '' and grasped_object_mask header seq: 106900 +stamp: + secs: 1735696260 + nsecs: 139355183 +frame_id: "camera_color_optical_frame" +[INFO] [1735696260.587567]: received mask_obs with header seq: 627 +stamp: + secs: 1735696260 + nsecs: 589304685 +frame_id: '' and grasped_object_mask header seq: 106906 +stamp: + secs: 1735696260 + nsecs: 338931322 +frame_id: "camera_color_optical_frame" +[INFO] [1735696260.789771]: received mask_obs with header seq: 628 +stamp: + secs: 1735696260 + nsecs: 791482925 +frame_id: '' and grasped_object_mask header seq: 106912 +stamp: + secs: 1735696260 + nsecs: 539266348 +frame_id: "camera_color_optical_frame" +[INFO] [1735696260.985334]: received mask_obs with header seq: 629 +stamp: + secs: 1735696260 + nsecs: 987113475 +frame_id: '' and grasped_object_mask header seq: 106918 +stamp: + secs: 1735696260 + nsecs: 740204811 +frame_id: "camera_color_optical_frame" +[INFO] [1735696261.188735]: received mask_obs with header seq: 630 +stamp: + secs: 1735696261 + nsecs: 190455913 +frame_id: '' and grasped_object_mask header seq: 106924 +stamp: + secs: 1735696260 + nsecs: 941231012 +frame_id: "camera_color_optical_frame" +[INFO] [1735696261.389721]: received mask_obs with header seq: 631 +stamp: + secs: 1735696261 + nsecs: 391401767 +frame_id: '' and grasped_object_mask header seq: 106930 +stamp: + secs: 1735696261 + nsecs: 142343044 +frame_id: "camera_color_optical_frame" +[INFO] [1735696261.585782]: received mask_obs with header seq: 632 +stamp: + secs: 1735696261 + nsecs: 587531089 +frame_id: '' and grasped_object_mask header seq: 106936 +stamp: + secs: 1735696261 + nsecs: 343541622 +frame_id: "camera_color_optical_frame" +[INFO] [1735696261.789434]: received mask_obs with header seq: 633 +stamp: + secs: 1735696261 + nsecs: 790921688 +frame_id: '' and grasped_object_mask header seq: 106942 +stamp: + secs: 1735696261 + nsecs: 543972015 +frame_id: "camera_color_optical_frame" +[INFO] [1735696261.987241]: received mask_obs with header seq: 634 +stamp: + secs: 1735696261 + nsecs: 988952636 +frame_id: '' and grasped_object_mask header seq: 106948 +stamp: + secs: 1735696261 + nsecs: 743818045 +frame_id: "camera_color_optical_frame" +[INFO] [1735696262.191002]: received mask_obs with header seq: 635 +stamp: + secs: 1735696262 + nsecs: 192732334 +frame_id: '' and grasped_object_mask header seq: 106954 +stamp: + secs: 1735696261 + nsecs: 943664551 +frame_id: "camera_color_optical_frame" +[INFO] [1735696262.403256]: received mask_obs with header seq: 636 +stamp: + secs: 1735696262 + nsecs: 404402494 +frame_id: '' and grasped_object_mask header seq: 106960 +stamp: + secs: 1735696262 + nsecs: 143512487 +frame_id: "camera_color_optical_frame" +[INFO] [1735696262.604522]: received mask_obs with header seq: 637 +stamp: + secs: 1735696262 + nsecs: 605961799 +frame_id: '' and grasped_object_mask header seq: 106966 +stamp: + secs: 1735696262 + nsecs: 343359947 +frame_id: "camera_color_optical_frame" +[INFO] [1735696262.799478]: received mask_obs with header seq: 638 +stamp: + secs: 1735696262 + nsecs: 801163434 +frame_id: '' and grasped_object_mask header seq: 106973 +stamp: + secs: 1735696262 + nsecs: 576503992 +frame_id: "camera_color_optical_frame" +[INFO] [1735696262.999594]: received mask_obs with header seq: 639 +stamp: + secs: 1735696263 + nsecs: 1360654 +frame_id: '' and grasped_object_mask header seq: 106979 +stamp: + secs: 1735696262 + nsecs: 776336432 +frame_id: "camera_color_optical_frame" +[INFO] [1735696263.200069]: received mask_obs with header seq: 640 +stamp: + secs: 1735696263 + nsecs: 201770067 +frame_id: '' and grasped_object_mask header seq: 106985 +stamp: + secs: 1735696262 + nsecs: 976167679 +frame_id: "camera_color_optical_frame" +[INFO] [1735696263.399051]: received mask_obs with header seq: 641 +stamp: + secs: 1735696263 + nsecs: 400734901 +frame_id: '' and grasped_object_mask header seq: 106991 +stamp: + secs: 1735696263 + nsecs: 175998688 +frame_id: "camera_color_optical_frame" +[INFO] [1735696263.598337]: received mask_obs with header seq: 642 +stamp: + secs: 1735696263 + nsecs: 600040674 +frame_id: '' and grasped_object_mask header seq: 106997 +stamp: + secs: 1735696263 + nsecs: 375829697 +frame_id: "camera_color_optical_frame" +[INFO] [1735696263.796250]: received mask_obs with header seq: 643 +stamp: + secs: 1735696263 + nsecs: 797915220 +frame_id: '' and grasped_object_mask header seq: 107002 +stamp: + secs: 1735696263 + nsecs: 542361259 +frame_id: "camera_color_optical_frame" +[INFO] [1735696263.992845]: received mask_obs with header seq: 644 +stamp: + secs: 1735696263 + nsecs: 994317293 +frame_id: '' and grasped_object_mask header seq: 107009 +stamp: + secs: 1735696263 + nsecs: 775510550 +frame_id: "camera_color_optical_frame" +[INFO] [1735696264.202031]: received mask_obs with header seq: 645 +stamp: + secs: 1735696264 + nsecs: 203741073 +frame_id: '' and grasped_object_mask header seq: 107015 +stamp: + secs: 1735696263 + nsecs: 975354195 +frame_id: "camera_color_optical_frame" +[INFO] [1735696264.404396]: received mask_obs with header seq: 646 +stamp: + secs: 1735696264 + nsecs: 406160831 +frame_id: '' and grasped_object_mask header seq: 107021 +stamp: + secs: 1735696264 + nsecs: 175197363 +frame_id: "camera_color_optical_frame" +[INFO] [1735696264.611541]: received mask_obs with header seq: 647 +stamp: + secs: 1735696264 + nsecs: 613259315 +frame_id: '' and grasped_object_mask header seq: 107027 +stamp: + secs: 1735696264 + nsecs: 375040531 +frame_id: "camera_color_optical_frame" +[INFO] [1735696264.807196]: received mask_obs with header seq: 648 +stamp: + secs: 1735696264 + nsecs: 808900833 +frame_id: '' and grasped_object_mask header seq: 107033 +stamp: + secs: 1735696264 + nsecs: 574879169 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.005336]: received mask_obs with header seq: 649 +stamp: + secs: 1735696265 + nsecs: 6977796 +frame_id: '' and grasped_object_mask header seq: 107039 +stamp: + secs: 1735696264 + nsecs: 774715662 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.204914]: received mask_obs with header seq: 650 +stamp: + secs: 1735696265 + nsecs: 206606149 +frame_id: '' and grasped_object_mask header seq: 107045 +stamp: + secs: 1735696264 + nsecs: 974551916 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.402736]: received mask_obs with header seq: 651 +stamp: + secs: 1735696265 + nsecs: 404468297 +frame_id: '' and grasped_object_mask header seq: 107051 +stamp: + secs: 1735696265 + nsecs: 174389124 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.606081]: received mask_obs with header seq: 652 +stamp: + secs: 1735696265 + nsecs: 607877016 +frame_id: '' and grasped_object_mask header seq: 107057 +stamp: + secs: 1735696265 + nsecs: 374225378 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.801584]: received mask_obs with header seq: 653 +stamp: + secs: 1735696265 + nsecs: 802728414 +frame_id: '' and grasped_object_mask header seq: 107063 +stamp: + secs: 1735696265 + nsecs: 574061871 +frame_id: "camera_color_optical_frame" +[INFO] [1735696265.999485]: received mask_obs with header seq: 654 +stamp: + secs: 1735696266 + nsecs: 826358 +frame_id: '' and grasped_object_mask header seq: 107069 +stamp: + secs: 1735696265 + nsecs: 773898125 +frame_id: "camera_color_optical_frame" +[INFO] [1735696266.199092]: received mask_obs with header seq: 655 +stamp: + secs: 1735696266 + nsecs: 200869321 +frame_id: '' and grasped_object_mask header seq: 107075 +stamp: + secs: 1735696265 + nsecs: 973734379 +frame_id: "camera_color_optical_frame" +[INFO] [1735696266.395631]: received mask_obs with header seq: 656 +stamp: + secs: 1735696266 + nsecs: 397406816 +frame_id: '' and grasped_object_mask header seq: 107081 +stamp: + secs: 1735696266 + nsecs: 173570871 +frame_id: "camera_color_optical_frame" +[INFO] [1735696266.603220]: received mask_obs with header seq: 657 +stamp: + secs: 1735696266 + nsecs: 604619503 +frame_id: '' and grasped_object_mask header seq: 107087 +stamp: + secs: 1735696266 + nsecs: 373407841 +frame_id: "camera_color_optical_frame" +[INFO] [1735696266.806563]: received mask_obs with header seq: 658 +stamp: + secs: 1735696266 + nsecs: 808249950 +frame_id: '' and grasped_object_mask header seq: 107093 +stamp: + secs: 1735696266 + nsecs: 573245764 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.000580]: received mask_obs with header seq: 659 +stamp: + secs: 1735696267 + nsecs: 2331495 +frame_id: '' and grasped_object_mask header seq: 107099 +stamp: + secs: 1735696266 + nsecs: 773084402 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.199214]: received mask_obs with header seq: 660 +stamp: + secs: 1735696267 + nsecs: 201011896 +frame_id: '' and grasped_object_mask header seq: 107105 +stamp: + secs: 1735696266 + nsecs: 972923279 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.397671]: received mask_obs with header seq: 661 +stamp: + secs: 1735696267 + nsecs: 399255990 +frame_id: '' and grasped_object_mask header seq: 107111 +stamp: + secs: 1735696267 + nsecs: 172762156 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.597920]: received mask_obs with header seq: 662 +stamp: + secs: 1735696267 + nsecs: 599429130 +frame_id: '' and grasped_object_mask header seq: 107117 +stamp: + secs: 1735696267 + nsecs: 372601032 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.801921]: received mask_obs with header seq: 663 +stamp: + secs: 1735696267 + nsecs: 803631544 +frame_id: '' and grasped_object_mask header seq: 107123 +stamp: + secs: 1735696267 + nsecs: 572439432 +frame_id: "camera_color_optical_frame" +[INFO] [1735696267.999267]: received mask_obs with header seq: 664 +stamp: + secs: 1735696268 + nsecs: 1058816 +frame_id: '' and grasped_object_mask header seq: 107129 +stamp: + secs: 1735696267 + nsecs: 772278547 +frame_id: "camera_color_optical_frame" +[INFO] [1735696268.054911]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1735696268.202532]: received mask_obs with header seq: 665 +stamp: + secs: 1735696268 + nsecs: 202919006 +frame_id: '' and grasped_object_mask header seq: 107135 +stamp: + secs: 1735696267 + nsecs: 972116470 +frame_id: "camera_color_optical_frame" +[INFO] [1735696268.944580]: Stored episode 5. +[INFO] [1735696268.953343]: Saving video... +[INFO] [1735696269.299867]: Video saved! +[INFO] [1735696269.343210]: resetting environment +[INFO] [1735696269.364842]: cleared current plan +[INFO] [1735696269.365075]: moving to home +[INFO] [1735696272.967643]: reached home +[INFO] [1735696272.969667]: reset action history +[INFO] [1735696275.863616]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Error executing job with overrides: ['agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'] +Traceback (most recent call last): + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 592, in main + workspace.eval() + File "/home/leonmkim/fish_leon/FISH/eval_robot.py", line 468, in eval + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + File "/home/leonmkim/.pyenv/versions/lerobot/lib/python3.10/site-packages/wandb/data_types.py", line 218, in __init__ + self._init_from_list(data, columns, optional, dtype) + File "/home/leonmkim/.pyenv/versions/lerobot/lib/python3.10/site-packages/wandb/data_types.py", line 243, in _init_from_list + self.add_data(*row) +TypeError: wandb.data_types.Table.add_data() argument after * must be an iterable, not float +Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace. +program exiting gracefully +proceeding to start episode! +[INFO] [1735696909.841831]: Saving video... +[INFO] [1735696909.842013]: Video saved! \ No newline at end of file diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3ffefd8852fc3e3f1a458e5ec049e59de6773b11 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-01T01:46:23.566585", + "startedAt": "2025-01-01T01:46:23.167015", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "284407f87cd673789a10dbd8e4d4f41dcad30ded" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.963, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.968, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.993, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 488.8013000488281 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..971f3ab51fdd800aea1fb49d815443bf458494b0 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "94c4871aa0a38b24cd6b714c4d872f1bdf15ee9e1368eca5c4c5cca7795db545", "size": 1147449, "path": "media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4"}, "global_step": 665, "_timestamp": 1735696303.6406088, "_runtime": 320.46008372306824, "_step": 10, "eval/1_eval": {"_type": "video-file", "sha256": "f99c821ca0f3dcf058d7bdc57f31c99fec024325049b678d22e60cdfcb7032ce", "size": 1154686, "path": "media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "497369a7fef0c32f3ee5bafccfc53b7a3889416759469d55aac8abeb7d3c5011", "size": 1153780, "path": "media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "0f8beb81bb83090b35f94094b007cb72a5ac5e21914596e2821215154f3398a6", "size": 1163442, "path": "media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d8469a9945cf9312a1f45cb2c0e3bef7b03b0c9263ac88eff33e692616036151", "size": 1242957, "path": "media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4"}, "eval/video": {"_type": "video-file", "sha256": "d8469a9945cf9312a1f45cb2c0e3bef7b03b0c9263ac88eff33e692616036151", "size": 1242957, "path": "media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4"}, "_wandb": {"runtime": 926}} \ No newline at end of file diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug-internal.log b/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2780c8030150722974220e42c7507589b190f143 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug-internal.log @@ -0,0 +1,1547 @@ +2024-12-31 20:46:23,179 INFO StreamThr :1737239 [internal.py:wandb_internal():85] W&B internal server running at pid: 1737239, started at: 2024-12-31 20:46:23.178761 +2024-12-31 20:46:23,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status +2024-12-31 20:46:23,197 INFO WriterThread:1737239 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb +2024-12-31 20:46:23,197 DEBUG SenderThread:1737239 [sender.py:send():379] send: header +2024-12-31 20:46:23,205 DEBUG SenderThread:1737239 [sender.py:send():379] send: run +2024-12-31 20:46:23,451 INFO SenderThread:1737239 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files +2024-12-31 20:46:23,452 INFO SenderThread:1737239 [sender.py:_start_run_threads():1188] run started: kccehv5k with start time 1735695983.180525 +2024-12-31 20:46:23,475 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 20:46:23,475 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: check_version +2024-12-31 20:46:23,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 20:46:23,553 DEBUG HandlerThread:1737239 [system_info.py:__init__():26] System info init +2024-12-31 20:46:23,554 DEBUG HandlerThread:1737239 [system_info.py:__init__():41] System info init done +2024-12-31 20:46:23,554 INFO HandlerThread:1737239 [system_monitor.py:start():194] Starting system monitor +2024-12-31 20:46:23,554 INFO SystemMonitor:1737239 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 20:46:23,554 INFO HandlerThread:1737239 [system_monitor.py:probe():214] Collecting system info +2024-12-31 20:46:23,555 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 20:46:23,555 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started disk monitoring +2024-12-31 20:46:23,556 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 20:46:23,556 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started memory monitoring +2024-12-31 20:46:23,557 INFO SystemMonitor:1737239 [interfaces.py:start():188] Started network monitoring +2024-12-31 20:46:23,566 DEBUG HandlerThread:1737239 [system_info.py:probe():152] Probing system +2024-12-31 20:46:23,567 DEBUG HandlerThread:1737239 [system_info.py:_probe_git():137] Probing git +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_info.py:_probe_git():145] Probing git done +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_info.py:probe():200] Probing system done +2024-12-31 20:46:23,572 DEBUG HandlerThread:1737239 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T01:46:23.566585', 'startedAt': '2025-01-01T01:46:23.167015', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '284407f87cd673789a10dbd8e4d4f41dcad30ded'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 4499.993, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 488.8013000488281}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 20:46:23,572 INFO HandlerThread:1737239 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 20:46:23,572 INFO HandlerThread:1737239 [system_monitor.py:probe():227] Publishing system info +2024-12-31 20:46:23,573 DEBUG HandlerThread:1737239 [system_info.py:_save_code():44] Saving code +2024-12-31 20:46:23,574 DEBUG HandlerThread:1737239 [system_info.py:_save_code():67] Saving code done +2024-12-31 20:46:23,574 DEBUG HandlerThread:1737239 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 20:46:23,637 DEBUG HandlerThread:1737239 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 20:46:23,638 INFO HandlerThread:1737239 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 20:46:23,645 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 20:46:23,645 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 20:46:23,914 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 20:46:23,915 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: python_packages +2024-12-31 20:46:23,916 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 20:46:23,916 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: stop_status +2024-12-31 20:46:23,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:23,918 INFO HandlerThread:1737239 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/tb" + save: true +} +control { + relay_id: "5f4965b8c2e34f0181637e6010b99766" +} +_info { + stream_id: "kccehv5k" +} + +2024-12-31 20:46:23,919 DEBUG HandlerThread:1737239 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 20:46:23,920 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/6wde7y2z-wandb-metadata.json +2024-12-31 20:46:23,935 INFO wandb-upload_2:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/thrz3lhd-diff.patch +2024-12-31 20:46:23,960 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/gf5ock7d-code/FISH/eval_robot.py +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:46:24,453 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code +2024-12-31 20:46:24,454 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH +2024-12-31 20:46:24,670 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/z7vm580p-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:46:24,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:25,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:26,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:27,917 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:28,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:29,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:30,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:31,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:32,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:33,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:34,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:35,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:36,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:37,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:38,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:39,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:40,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:41,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:42,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:43,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:44,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:45,314 DEBUG SenderThread:1737239 [sender.py:send():379] send: telemetry +2024-12-31 20:46:45,314 DEBUG SenderThread:1737239 [sender.py:send():379] send: tbrecord +2024-12-31 20:46:45,314 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:45,315 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:46:45,315 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 with policy live +2024-12-31 20:46:45,315 DEBUG SenderThread:1737239 [sender.py:send():379] send: config +2024-12-31 20:46:45,457 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:45,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:46,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:47,458 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:47,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:48,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:49,459 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:49,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:50,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:51,197 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:51,459 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:46:51,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:52,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:53,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:54,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:55,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:56,201 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:46:56,461 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 20:46:56,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:57,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:58,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:46:59,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:00,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:01,318 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:01,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:02,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:03,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:04,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:05,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:06,319 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:06,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:07,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:08,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:09,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:10,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:11,320 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:11,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:12,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:13,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:14,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:15,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:16,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:16,924 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:17,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:18,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:19,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:20,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:21,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:21,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:22,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:23,557 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 20:47:23,558 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:47:23,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:24,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:25,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:26,558 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:26,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:27,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:28,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:29,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:30,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:31,559 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:31,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:32,927 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:33,471 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:33,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:34,927 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:35,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:36,715 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:36,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:37,472 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:37,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:38,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:39,472 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:39,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:40,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:41,473 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:41,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:41,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:42,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:43,473 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:43,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:44,928 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:45,474 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:45,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:46,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:46,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:47,475 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:47,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:48,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:49,475 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:49,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:50,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:51,476 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:51,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:52,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:52,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:53,477 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:53,558 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:47:53,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:54,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:55,477 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:55,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:56,929 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:57,141 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:47:57,478 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:57,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:58,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:47:59,478 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:47:59,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:00,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:01,479 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:01,930 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:02,145 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:02,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:03,321 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:48:03,321 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 with policy now +2024-12-31 20:48:03,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:48:03,322 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:48:03,322 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:48:03,323 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos +2024-12-31 20:48:03,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:48:03,576 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/bfcifyvh-media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,840 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/5sr4pn21-media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 20:48:03,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:04,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:05,480 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:05,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:06,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:07,481 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:07,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:07,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:08,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:09,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:10,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:11,482 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:48:11,483 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:11,761 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/pulmzem4-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:48:11,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:12,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:13,483 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:13,509 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:13,933 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:14,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:15,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:16,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:17,484 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:17,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:18,698 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:18,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:19,485 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:19,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:20,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:21,486 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:21,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:22,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:23,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:48:23,894 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:23,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:24,487 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:24,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:25,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:26,488 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:26,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:27,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:28,488 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:28,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:29,092 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:29,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:30,489 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:30,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:31,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:32,490 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:32,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:33,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:34,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:34,490 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:34,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:35,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:36,491 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:36,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:37,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:38,492 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:38,936 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:39,487 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:39,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:40,493 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:40,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:41,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:42,493 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:42,937 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:43,161 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:48:43,161 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:48:43,161 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 with policy now +2024-12-31 20:48:43,162 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:48:43,162 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:48:43,162 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:48:43,424 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/rhfg4oqz-media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:48:43,760 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/jq399vyr-media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 20:48:43,947 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:44,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:44,941 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:45,165 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:45,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:46,494 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:46,947 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:47,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:48,495 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:48,944 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:49,941 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:50,589 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:50,945 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:51,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:52,496 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:48:52,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:53,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:48:53,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:54,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:55,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:56,560 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:48:56,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:57,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:58,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:48:59,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:00,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:01,560 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:01,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:02,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:03,374 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:03,374 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:03,375 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:03,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:03,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:03,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:04,500 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:04,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:05,960 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:06,501 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:06,792 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:06,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:07,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:08,502 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:08,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:09,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:10,503 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:10,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:11,925 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:11,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:12,503 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:12,956 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:13,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:14,504 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:14,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:15,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:16,505 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:16,926 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:16,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:17,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:18,505 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:18,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:19,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:20,506 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:20,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:21,935 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:21,957 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:22,507 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:22,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:23,559 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:49:23,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:24,508 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:24,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:25,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:26,509 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:26,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:26,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:27,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:28,510 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:28,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:29,958 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:30,511 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:30,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:31,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:32,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:32,512 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:32,959 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:33,962 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:34,512 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:34,798 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:49:34,798 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 with policy now +2024-12-31 20:49:34,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:49:34,802 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:34,802 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:34,802 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:34,963 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:35,093 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/ax5t9rrt-media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:49:35,788 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/136e1m9w-media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 20:49:35,968 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:36,513 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:36,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:37,824 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:37,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:38,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:39,970 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:40,514 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:40,969 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:41,968 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:42,515 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:42,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:43,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:43,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:44,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:46,005 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:46,216 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:49:46,216 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:49:46,217 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:49:46,516 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:46,517 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:49:46,774 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/xkps3yje-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:49:46,990 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:47,987 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:48,517 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:48,984 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:49,073 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:49,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:50,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:51,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:52,518 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:52,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:53,560 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:49:53,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:54,182 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:54,519 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:54,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:55,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:56,520 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:56,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:57,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:58,521 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:49:58,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:49:59,372 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:49:59,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:00,522 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:00,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:01,978 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:02,522 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:02,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:03,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:04,384 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:04,523 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:04,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:05,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:06,524 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:06,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:07,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:08,525 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:08,979 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:09,578 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:09,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:10,525 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:10,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:11,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:12,526 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:12,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:13,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:14,527 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:14,772 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:14,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:15,981 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:16,528 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:16,986 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:17,939 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:50:17,939 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 with policy now +2024-12-31 20:50:17,946 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:50:17,946 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:50:17,946 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:50:17,947 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:50:17,984 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:18,196 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/v104jt0i-media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,528 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:50:18,805 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/6q36fotf-media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 20:50:18,987 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:19,977 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:19,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:20,529 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:20,991 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:21,990 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:22,991 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:23,560 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:50:23,995 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:24,531 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:24,998 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:25,505 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:25,999 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:26,531 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:27,004 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:28,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:28,994 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:29,996 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:30,506 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:31,000 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:32,017 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:33,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:34,003 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:35,021 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:35,276 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:50:35,276 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:50:35,277 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:50:35,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:50:35,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:50:35,998 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:36,063 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:36,534 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:37,019 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:38,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:38,535 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:39,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:40,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:40,536 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:41,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:41,160 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:42,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:42,537 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:43,010 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:44,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:44,537 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:45,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:46,011 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:46,198 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:46,538 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:47,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:48,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:48,539 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:49,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:50,012 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:50,540 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:51,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:51,389 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:52,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:52,540 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:53,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:53,561 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:50:54,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:54,541 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:55,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:56,013 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:56,542 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:56,589 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:50:57,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:58,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:50:58,543 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:50:59,014 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:00,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:00,544 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:01,015 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:01,790 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:02,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:02,544 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:03,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:04,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:04,545 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:05,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:06,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:06,546 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:06,807 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:07,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:08,016 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:08,546 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:09,023 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:09,345 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 20:51:09,345 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 20:51:09,345 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 with policy now +2024-12-31 20:51:09,346 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:51:09,346 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:51:09,346 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:09,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 20:51:09,645 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/ge0ui9g0-media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:09,912 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/g212tycz-media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 20:51:10,020 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:10,547 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:11,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:12,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:12,365 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:13,030 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:14,035 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:14,549 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:15,032 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:16,046 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:17,045 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:17,909 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:18,046 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:18,550 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:19,056 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:20,083 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:21,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:22,054 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:22,910 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:23,056 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:23,561 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:51:24,059 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:25,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:26,066 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:27,065 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:28,061 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:28,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:29,068 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:30,068 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:31,071 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:32,079 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:33,099 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:33,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:34,083 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:35,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:36,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:37,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:38,091 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:38,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:39,100 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:40,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:41,091 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:42,096 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:43,099 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:43,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:44,112 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:44,361 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 20:51:44,361 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 20:51:44,361 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 20:51:44,559 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:44,810 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/gknw2j08-events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 20:51:45,108 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:46,115 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:46,560 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 20:51:47,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:48,123 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:49,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:49,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:50,129 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:51,122 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:52,137 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:53,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:53,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:51:54,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:54,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:51:55,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:56,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:57,132 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:58,141 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:59,144 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:51:59,562 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:00,149 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:01,144 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:02,158 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:03,157 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:04,159 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:04,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:05,160 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:06,155 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:07,159 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:08,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:09,163 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:09,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:10,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:11,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:12,179 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:13,177 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:14,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:14,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:15,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:16,173 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:17,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:18,181 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:19,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:19,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:20,184 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:21,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:22,192 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:23,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:23,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:52:24,198 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:25,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:25,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:26,201 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:27,207 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:28,199 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:29,204 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:30,207 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:30,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:31,209 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:32,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:33,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:34,208 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:35,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:35,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:36,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:37,208 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:38,216 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:39,217 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:40,219 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:40,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:41,221 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:42,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:43,220 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:44,221 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:45,230 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:45,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:46,229 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:47,246 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:48,234 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:49,241 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:50,241 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:50,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:51,242 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:52,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:53,243 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:53,562 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:52:54,249 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:55,252 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:56,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:56,563 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:52:57,245 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:58,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:52:59,248 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:00,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:01,258 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:01,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:02,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:03,252 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:04,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:05,259 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:06,258 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:06,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:07,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:08,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:09,275 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:10,262 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:11,265 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:11,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:12,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:13,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:14,265 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:15,272 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:16,272 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:16,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:17,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:18,302 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:19,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:20,294 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:21,289 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:21,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:22,297 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:23,308 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:23,563 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:53:24,307 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:25,321 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:26,317 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:27,300 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:27,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:28,329 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:29,317 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:30,325 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:31,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:32,330 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:32,564 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:33,340 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:34,328 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:35,329 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:36,327 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:37,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:37,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:38,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:39,339 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:40,333 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:41,332 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:42,335 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:42,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:43,336 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:44,336 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:45,346 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:46,346 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:47,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:47,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:48,341 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:49,345 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:50,349 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:51,354 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:52,348 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:52,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:53,357 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:53,571 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:53:54,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:55,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:56,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:57,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:57,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:53:58,360 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:53:59,363 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:00,361 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:01,357 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:02,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:02,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:03,359 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:04,364 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:05,367 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:06,364 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:07,362 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:07,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:08,385 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:09,370 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:10,376 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:11,379 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:12,375 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:12,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:13,376 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:14,387 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:15,392 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:16,400 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:17,388 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:17,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:18,393 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:19,386 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:20,398 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:21,392 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:22,399 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:22,574 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:23,397 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:23,564 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:54:24,397 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:25,400 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:26,404 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:27,420 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:28,399 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:28,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:29,405 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:30,402 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:31,409 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:32,413 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:33,421 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:33,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:34,426 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:35,430 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:36,429 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:37,424 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:38,426 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:38,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:39,428 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:40,433 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:41,447 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:42,449 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:43,447 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:43,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:44,452 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:45,455 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:46,444 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:47,451 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:48,450 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:48,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:49,451 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:50,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:51,464 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:52,470 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:53,453 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:53,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:54:54,461 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:54,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:54:55,465 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:56,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:57,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:58,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:59,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:54:59,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:00,483 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:01,476 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:02,484 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:03,475 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:04,482 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:04,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:05,479 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:06,487 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:07,492 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:08,488 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:09,503 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:09,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:10,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:11,501 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:12,492 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:13,495 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:14,497 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:14,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:15,494 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:16,494 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:17,496 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:18,501 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:19,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:19,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:20,518 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:21,527 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:22,522 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:23,513 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:23,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:55:24,510 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:25,516 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:25,565 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:26,516 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:27,521 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:28,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:29,520 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:30,538 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:30,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:31,528 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:32,533 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:33,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:34,530 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:35,526 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:35,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:36,536 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:37,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:38,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:39,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:40,544 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:40,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:41,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:42,535 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:43,542 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:44,549 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:45,545 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:45,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:46,548 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:47,556 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:48,539 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:49,556 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:50,549 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:50,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:51,557 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:52,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:53,565 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:55:53,574 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:54,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:55,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:56,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:55:56,582 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:57,576 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:58,578 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:55:59,584 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:00,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:01,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:01,582 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:02,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:03,588 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:04,585 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:05,597 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:06,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:06,595 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:07,590 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:08,593 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:09,594 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:10,595 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:11,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:11,613 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:12,601 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:13,618 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:14,618 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:15,616 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:16,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:16,606 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:17,621 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:18,620 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:19,622 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:20,628 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:21,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:21,634 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:22,629 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:23,566 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:56:23,634 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:24,638 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:25,642 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:26,641 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:27,566 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:27,642 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:28,659 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:29,657 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:30,644 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:31,641 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:32,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:32,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:33,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:34,654 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:35,651 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:36,653 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:37,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:37,651 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:38,663 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:39,665 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:40,659 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:41,658 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:42,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:42,658 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:43,684 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:44,674 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:45,663 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:46,669 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:47,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:47,676 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:48,681 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:49,691 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:50,690 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:51,694 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:52,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:52,697 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:53,566 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:56:53,707 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:54,696 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:55,695 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:56,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:57,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:58,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:56:58,722 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:56:59,713 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:00,710 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:01,709 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:02,714 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:03,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:03,741 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:04,740 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:05,739 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:06,739 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:07,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:08,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:08,742 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:09,749 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:10,748 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:11,742 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:12,763 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:13,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:13,746 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:14,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:15,748 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:16,745 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:17,752 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:18,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:18,749 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:19,754 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:20,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:21,761 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:22,772 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:23,567 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:57:23,763 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:24,567 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:24,775 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:25,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:26,760 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:27,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:28,762 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:29,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:29,781 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:30,778 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:31,771 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:32,777 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:33,774 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:34,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:34,767 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:35,770 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:36,778 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:37,777 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:38,780 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:39,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:39,781 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:40,788 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:41,785 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:42,795 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:43,794 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:44,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:44,800 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:45,800 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:46,799 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:47,808 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:48,801 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:49,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:49,807 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:50,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:51,811 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:52,825 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:53,567 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:57:53,805 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:54,811 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:55,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:57:55,808 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:56,806 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:57,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:58,812 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:57:59,816 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:00,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:00,814 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:01,818 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:02,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:03,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:04,828 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:05,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:05,822 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:06,823 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:07,828 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:08,825 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:09,831 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:10,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:10,831 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:11,839 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:12,849 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:13,851 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:14,846 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:15,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:15,842 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:16,851 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:17,849 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:18,860 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:19,854 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:20,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:20,850 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:21,854 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:22,855 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:23,568 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:58:23,865 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:24,869 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:25,870 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:26,568 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:26,871 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:27,871 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:28,874 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:29,875 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:30,879 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:31,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:31,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:32,887 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:33,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:34,891 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:35,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:36,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:36,890 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:37,893 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:38,892 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:39,898 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:40,909 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:41,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:41,922 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:42,901 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:43,921 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:44,907 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:45,910 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:46,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:46,913 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:47,915 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:48,920 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:49,914 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:50,912 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:51,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:51,919 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:52,918 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:53,568 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:58:53,915 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:54,931 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:55,923 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:56,932 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:57,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:58:57,942 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:58,944 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:58:59,934 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:00,954 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:01,948 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:02,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:02,952 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:03,950 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:04,954 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:05,955 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:06,971 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:07,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:07,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:08,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:09,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:10,973 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:11,982 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:12,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:12,982 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:13,980 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:14,985 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:15,993 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:16,999 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:17,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:18,007 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:19,003 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:19,995 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:21,000 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:22,007 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:22,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:23,008 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:23,569 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:59:24,009 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:25,023 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:26,026 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:27,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:28,024 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:28,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:29,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:30,027 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:31,034 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:32,044 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:33,038 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:33,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:34,032 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:35,055 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:36,050 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:37,054 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:38,048 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:38,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:39,053 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:40,048 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:41,062 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:42,063 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:43,064 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:43,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:44,075 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:45,073 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:46,072 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:47,075 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:48,092 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:48,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:49,078 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:50,080 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:51,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:52,085 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:53,090 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:53,569 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 20:59:54,089 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:54,569 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 20:59:55,095 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:56,102 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:57,120 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:58,107 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:59,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 20:59:59,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:00,127 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:01,116 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:02,131 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:03,113 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:04,136 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:04,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:05,117 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:06,122 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:07,121 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:08,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:09,126 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:09,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:10,124 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:11,125 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:12,134 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:13,135 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:14,133 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:14,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:15,138 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:16,137 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:17,143 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:18,150 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:19,148 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:19,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:20,149 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:21,152 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:22,163 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:23,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:23,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:00:24,178 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:25,166 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:25,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:26,175 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:27,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:28,172 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:29,171 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:30,169 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:30,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:31,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:32,175 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:33,179 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:34,176 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:35,190 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:35,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:36,180 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:37,204 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:38,182 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:39,194 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:40,194 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:40,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:41,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:42,192 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:43,183 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:44,189 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:45,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:45,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:46,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:47,190 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:48,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:49,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:50,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:50,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:51,195 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:52,193 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:53,196 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:53,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:00:54,224 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:55,212 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:56,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:56,570 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:00:57,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:58,209 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:00:59,213 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:00,215 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:01,215 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:01,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:02,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:03,219 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:04,223 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:05,222 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:06,223 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:06,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:07,225 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:08,233 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:09,235 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:10,237 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:11,243 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:11,572 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:12,244 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:13,245 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:14,244 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:15,250 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:16,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:16,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:17,256 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:18,264 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:19,264 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:20,266 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:21,273 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:21,573 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:22,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:23,271 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:23,570 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:01:24,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:25,267 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:26,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:27,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:27,571 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:28,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:29,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:30,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:30,766 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:31,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:32,268 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:32,686 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:33,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:34,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:35,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:36,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:37,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:37,686 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:38,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:39,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:40,269 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:41,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:42,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:42,687 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:43,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:44,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:45,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:46,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:47,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:47,688 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:48,270 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:49,271 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:01:49,889 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 21:01:49,889 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send():379] send: files +2024-12-31 21:01:49,890 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 with policy now +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 21:01:49,890 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:01:49,891 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:49,891 DEBUG SenderThread:1737239 [sender.py:send():379] send: telemetry +2024-12-31 21:01:49,891 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: server_info +2024-12-31 21:01:49,973 DEBUG SenderThread:1737239 [sender.py:send():379] send: exit +2024-12-31 21:01:49,973 INFO SenderThread:1737239 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 21:01:49,973 INFO SenderThread:1737239 [sender.py:send_exit():588] handling runtime: 926 +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:send_exit():594] send defer +2024-12-31 21:01:49,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,974 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 21:01:49,974 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 21:01:49,974 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 1 +2024-12-31 21:01:49,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,974 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 21:01:49,975 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:49,975 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 21:01:49,975 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 2 +2024-12-31 21:01:49,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:49,975 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 21:01:49,975 INFO HandlerThread:1737239 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 21:01:49,975 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 21:01:49,975 DEBUG SystemMonitor:1737239 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 21:01:49,976 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 21:01:49,976 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 21:01:50,028 INFO HandlerThread:1737239 [interfaces.py:finish():200] Joined network monitor +2024-12-31 21:01:50,028 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 3 +2024-12-31 21:01:50,029 DEBUG SenderThread:1737239 [sender.py:send():379] send: stats +2024-12-31 21:01:50,029 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:50,029 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 21:01:50,029 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 21:01:50,029 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 4 +2024-12-31 21:01:50,029 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:50,029 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 21:01:50,169 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/qqawmlse-media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:50,773 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:50,774 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval +2024-12-31 21:01:51,039 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /tmp/tmp9bpkh5biwandb/h9z1qd6j-media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:01:55,416 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:55,416 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:55,417 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,417 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 21:01:55,417 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 5 +2024-12-31 21:01:55,417 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:01:55,417 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,418 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 21:01:55,418 DEBUG SenderThread:1737239 [sender.py:send():379] send: history +2024-12-31 21:01:55,418 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: summary_record +2024-12-31 21:01:55,419 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:55,419 DEBUG SenderThread:1737239 [sender.py:send():379] send: summary +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 21:01:55,423 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 21:01:55,423 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 6 +2024-12-31 21:01:55,424 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,424 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 21:01:55,424 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,424 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 21:01:55,427 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 21:01:55,553 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 7 +2024-12-31 21:01:55,554 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:55,554 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 21:01:55,554 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:55,554 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 21:01:55,776 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 21:01:55,777 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:01:55,974 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 21:01:55,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:58,233 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 8 +2024-12-31 21:01:58,233 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:01:58,233 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:58,234 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 21:01:58,234 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:58,234 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 21:01:58,234 INFO SenderThread:1737239 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 21:01:58,240 INFO SenderThread:1737239 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 21:01:58,254 INFO SenderThread:1737239 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 21:01:58,263 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 9 +2024-12-31 21:01:58,263 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:01:58,263 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 21:01:58,264 DEBUG SenderThread:1737239 [sender.py:send():379] send: artifact +2024-12-31 21:01:58,778 INFO Thread-12 :1737239 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:01:58,975 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:01:59,219 INFO wandb-upload_1:1737239 [upload_job.py:push():88] Uploaded file /tmp/tmpx0ryqqaf/wandb-job.json +2024-12-31 21:01:59,269 INFO wandb-upload_2:1737239 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpsqvtcw6o +2024-12-31 21:01:59,273 INFO wandb-upload_0:1737239 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmzppz3e_ +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDExMjAxNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA3MTYzNw==', 'versionIndex': 388}}} +2024-12-31 21:01:59,790 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 21:01:59,790 INFO SenderThread:1737239 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml config.yaml +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json wandb-summary.json +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt requirements.txt +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-metadata.json wandb-metadata.json +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 events.out.tfevents.1735695983.leonmkim-ROG-Strix-G15CS-G15CS.1737022.0 +2024-12-31 21:02:00,779 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log output.log +2024-12-31 21:02:00,780 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/diff.patch diff.patch +2024-12-31 21:02:00,784 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 media/videos/eval/video_5_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 media/videos/eval/4_eval_4_d8469a9945cf9312a1f4.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 media/videos/eval/1_eval_1_f99c821ca0f3dcf058d7.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 media/videos/eval/3_eval_3_0f8beb81bb83090b35f9.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 media/videos/eval/2_eval_2_497369a7fef0c32f3ee5.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 media/videos/eval/0_eval_0_94c4871aa0a38b24cd6b.mp4 +2024-12-31 21:02:00,786 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 10 +2024-12-31 21:02:00,786 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:00,787 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:00,787 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 21:02:00,787 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:00,787 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 21:02:00,787 INFO SenderThread:1737239 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:02:00,976 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:02:00,977 INFO wandb-upload_1:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/config.yaml +2024-12-31 21:02:00,977 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:00,978 INFO wandb-upload_0:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/requirements.txt +2024-12-31 21:02:00,990 INFO wandb-upload_2:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/wandb-summary.json +2024-12-31 21:02:01,082 INFO wandb-upload_3:1737239 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/files/output.log +2024-12-31 21:02:01,282 INFO Thread-11 (_thread_body):1737239 [sender.py:transition_state():614] send defer: 11 +2024-12-31 21:02:01,282 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,282 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 21:02:01,282 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,283 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 21:02:01,283 INFO SenderThread:1737239 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 12 +2024-12-31 21:02:01,284 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,284 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 21:02:01,284 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 21:02:01,284 INFO SenderThread:1737239 [file_stream.py:finish():601] file stream finish called +2024-12-31 21:02:01,457 INFO SenderThread:1737239 [file_stream.py:finish():605] file stream finish is done +2024-12-31 21:02:01,457 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 13 +2024-12-31 21:02:01,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,458 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 21:02:01,458 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,458 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 21:02:01,458 INFO SenderThread:1737239 [sender.py:transition_state():614] send defer: 14 +2024-12-31 21:02:01,458 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: defer +2024-12-31 21:02:01,458 INFO HandlerThread:1737239 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send():379] send: final +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send():379] send: footer +2024-12-31 21:02:01,459 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: defer +2024-12-31 21:02:01,459 INFO SenderThread:1737239 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 21:02:01,460 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 21:02:01,460 DEBUG SenderThread:1737239 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 21:02:01,461 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 21:02:01,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 21:02:01,471 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 21:02:01,472 DEBUG HandlerThread:1737239 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 21:02:01,472 INFO HandlerThread:1737239 [handler.py:finish():882] shutting down handler +2024-12-31 21:02:02,460 INFO WriterThread:1737239 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [sender.py:finish():1615] shutting down sender +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 21:02:02,460 INFO SenderThread:1737239 [file_pusher.py:join():175] waiting for file pusher diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug.log b/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3771954035b92f652620f179459bd09a33fa9283 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug.log @@ -0,0 +1,37 @@ +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Configure stats pid to 1737022 +2024-12-31 20:46:23,172 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/settings +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug.log +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/wandb/run-20241231_204623-kccehv5k/logs/debug-internal.log +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():569] calling init triggers +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k'} +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():619] starting backend +2024-12-31 20:46:23,173 INFO MainThread:1737022 [wandb_init.py:init():623] setting up manager +2024-12-31 20:46:23,178 INFO MainThread:1737022 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-31 20:46:23,180 INFO MainThread:1737022 [wandb_init.py:init():631] backend started and connected +2024-12-31 20:46:23,196 INFO MainThread:1737022 [wandb_init.py:init():720] updated telemetry +2024-12-31 20:46:23,202 INFO MainThread:1737022 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-31 20:46:23,474 INFO MainThread:1737022 [wandb_run.py:_on_init():2435] communicating current version +2024-12-31 20:46:23,539 INFO MainThread:1737022 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-31 20:46:23,540 INFO MainThread:1737022 [wandb_init.py:init():804] starting run threads in backend +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_console_start():2413] atexit reg +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-31 20:46:23,916 INFO MainThread:1737022 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-31 20:46:23,918 INFO MainThread:1737022 [wandb_init.py:init():847] run started, returning control to user process +2024-12-31 20:46:23,918 INFO MainThread:1737022 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/kccehv5k/tb, True +2024-12-31 20:46:42,413 INFO MainThread:1737022 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_finish():2107] finishing run serialexperimentsleon/extrinsic_contact_downstream/kccehv5k +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_atexit_cleanup():2374] got exitcode: 0 +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_restore():2352] restore +2024-12-31 21:01:49,852 INFO MainThread:1737022 [wandb_run.py:_restore():2358] restore done +2024-12-31 21:02:02,473 INFO MainThread:1737022 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 21:02:02,473 INFO MainThread:1737022 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 21:02:02,478 INFO MainThread:1737022 [wandb_run.py:_footer_sync_info():3975] logging synced files diff --git a/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb b/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1d3a90e81cd3bb7d9ed255c35c04e560c7695e78 --- /dev/null +++ b/kccehv5k/wandb/run-20241231_204623-kccehv5k/run-kccehv5k.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a0dd9e71abb22b7c36951ad51efb439f1c36def6563afbd44e0f545be3a1cc +size 312345 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..288662b7199473d9fd2691ec9cd6d993359eb188 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f95bac01e25e73c2338d48e07d9cc3c49365efbb1463bfbd6739b50fbf57086 +size 910102870 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..288662b7199473d9fd2691ec9cd6d993359eb188 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f95bac01e25e73c2338d48e07d9cc3c49365efbb1463bfbd6739b50fbf57086 +size 910102870 diff --git a/yyf8aq3v/episode_rosbags/episode_3_2025-01-07-17-39-11.bag b/yyf8aq3v/episode_rosbags/episode_3_2025-01-07-17-39-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..17cc7aaf2110dce38edce9d5679f6afd5ed06956 --- /dev/null +++ b/yyf8aq3v/episode_rosbags/episode_3_2025-01-07-17-39-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68f226626d34d20e3bd72dfacb4849329c44569a0995e27bd94f6f299bc6818e +size 1412657765