diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..f5bf42899862975682a0ab1645d94f33b18107e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,91 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_9_2025-01-17-14-54-53.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_18_2025-01-17-15-02-05.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_4_2025-01-17-14-51-15.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_19_2025-01-17-15-02-52.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_14_2025-01-17-14-58-54.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_7_2025-01-15-21-31-54.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_3_2025-01-17-14-50-30.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_10_2025-01-17-14-55-42.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_11_2025-01-17-14-56-29.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_0_2025-01-17-14-48-14.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_8_2025-01-17-14-54-08.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_1_2025-01-17-14-48-58.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_17_2025-01-17-15-01-18.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_2_2025-01-17-14-49-45.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_6_2025-01-17-14-52-43.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_15_2025-01-17-14-59-46.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_3_2025-01-15-21-29-12.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_5_2025-01-15-21-30-43.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_10_2025-01-15-21-34-09.bag filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_5_2025-01-17-14-51-58.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_4_2025-01-15-21-29-58.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_16_2025-01-15-21-38-58.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/episode_rosbags/episode_6_2025-01-15-21-31-20.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 filter=lfs diff=lfs merge=lfs -text +up07mbix/episode_rosbags/episode_16_2025-01-17-15-00-28.bag filter=lfs diff=lfs merge=lfs -text +vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vfeb54wi/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/7ncoq89x/.hydra/config.yaml b/7ncoq89x/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6218867bdef1ad3ae0c2d6cc61cdafef42eee2 --- /dev/null +++ b/7ncoq89x/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7ncoq89x/.hydra/hydra.yaml b/7ncoq89x/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa094e54bff3a79d799dca3deecd50d7b5b42e3e --- /dev/null +++ b/7ncoq89x/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/7ncoq89x + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7ncoq89x/.hydra/overrides.yaml b/7ncoq89x/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7ncoq89x/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7ncoq89x/eval_policy.log b/7ncoq89x/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..6790ca341fe3c8465059738f29545105f40e9a24 --- /dev/null +++ b/7ncoq89x/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-15 21:24:46,529][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-15 21:24:46,533][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-15 21:24:50,463][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2588cfcc30748bfcb0b9c0b081256fc528264762 --- /dev/null +++ b/config.yaml @@ -0,0 +1,573 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '3456_0' +wandb_notes: 3456_0_req_3921_0 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_1_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..0510d45917b8a78c7aafa86eb5b143a0092f2547 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95f464e0e376b1204aa40975f1154983fbc6d45332d048b70de3cce0e77dc7ac +size 909989782 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fadab97001154bdc8578784650c3144ec84aea9 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f858d04652bf8ee9890c569478f5f1b955b586adf6b334b48ef7eda1441df6f +size 909989782 diff --git a/snapshot_12999.pt b/snapshot_12999.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4026858ab42bb39c2b81e01b3c2158d2c3404cf --- /dev/null +++ b/snapshot_12999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:162b31d7c66d9c1cd548d0633e91129d46907f0672a5b77f335fa7057464004c +size 909989782 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a17640a2cd34baacffcc6b791e4c8c06f5dea87 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:215eafcab96d277df800a2fb29094f86af9e541f2d22b98e35816069e1a3367d +size 909989782 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fd70ccb2c9a648d953d1979ada5f93c22d2df4c --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438fa880711c10a6d73ba883b6a8722a86bbe4b3b9acd006ded7a4179ed2d77a +size 909989782 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f2eb700ed9739e5d3b2a57e159685cd9991267f --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd37a8f977b147260e8f63074751598826021a7531cc9f0517d3ae7e995c06e0 +size 909989782 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f57b0b0e4895d7f480f72df893f5ca168a448654 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df7d7d020e73dd1742111a5f870165f0a046f8c9ade138e40eae58e09f4f71b2 +size 909989782 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8543441044e5bace0a7fb0384174df1e7addc96c --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:066f056ce520f91de9334327b64ff56714cf384034f6bb37dc6cee20843eb8cd +size 909989782 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa380d454b9f6131e8a758a49fe5a471ff1c26c2 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4f41ae84816ffe7ca22a929c20ea533eef96e40b98e9e286d1e65a02d990857 +size 909989782 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..13c55cda0b908e2318f3fa340f19fc4c7c8d73d8 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874d97f963109073e450edb92faf0ba6987384295822d25c547d10e143da5fef +size 909989782 diff --git a/snapshot_7999.pt b/snapshot_7999.pt new file mode 100644 index 0000000000000000000000000000000000000000..b47f8baf681e4959d153e6ce2e12adc5bbba2c14 --- /dev/null +++ b/snapshot_7999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7514d579380340f84461ebd569d579b2d5150a147f78cbaca2a9f9c50e07b304 +size 909989782 diff --git a/snapshot_8749.pt b/snapshot_8749.pt new file mode 100644 index 0000000000000000000000000000000000000000..c566f7c0446c9164bdb4d64ad1bd22029a7ee76d --- /dev/null +++ b/snapshot_8749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:804f42535d167c82179ef9c4b695180b2eb7c74e124a1658c6575fd3b8906465 +size 909989782 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..adc11a107c2800d472cabd10140c951a523ddf95 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ea90d38f0aa5042b3d64c8fc2f6778a0fb7289a09ad07ded77392226b1f18f +size 909989782 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..f57b0b0e4895d7f480f72df893f5ca168a448654 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df7d7d020e73dd1742111a5f870165f0a046f8c9ade138e40eae58e09f4f71b2 +size 909989782 diff --git a/up07mbix/.hydra/config.yaml b/up07mbix/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c6218867bdef1ad3ae0c2d6cc61cdafef42eee2 --- /dev/null +++ b/up07mbix/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/up07mbix/.hydra/hydra.yaml b/up07mbix/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fb12fbfaf5a1f2869fd2a5e83d8533c1c4885ce --- /dev/null +++ b/up07mbix/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/up07mbix/.hydra/overrides.yaml b/up07mbix/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/up07mbix/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/up07mbix/episode_rosbags/aligned_depth_to_color_K.npy b/up07mbix/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/up07mbix/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/up07mbix/episode_rosbags/cam_tf_world.npy b/up07mbix/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/up07mbix/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/up07mbix/episode_rosbags/color_K.npy b/up07mbix/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/up07mbix/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/up07mbix/episode_rosbags/depth_K.npy b/up07mbix/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/up07mbix/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/up07mbix/episode_rosbags/episode_0_2025-01-17-14-48-14.bag b/up07mbix/episode_rosbags/episode_0_2025-01-17-14-48-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..6f9264d1fe1e0565c6110010464192e247a6950c --- /dev/null +++ b/up07mbix/episode_rosbags/episode_0_2025-01-17-14-48-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0214164b2204e250998344d2d88002cacd3ca44ed05e3eaecdc2db06d0f9b8e +size 1410704824 diff --git a/up07mbix/episode_rosbags/episode_10_2025-01-17-14-55-42.bag b/up07mbix/episode_rosbags/episode_10_2025-01-17-14-55-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..3e4a252910ab3ed6a51c6f0d1301f0503d8d6348 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_10_2025-01-17-14-55-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6733df8e5d5f1631b582838797c96a7acb021a86c95df1426503a2e7be097e54 +size 1409098682 diff --git a/up07mbix/episode_rosbags/episode_11_2025-01-17-14-56-29.bag b/up07mbix/episode_rosbags/episode_11_2025-01-17-14-56-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..d5af1123fb2e52de1331918251f64cbb9af33709 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_11_2025-01-17-14-56-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9937803aee9e2b7917bfefd37c095a0e7432dfc7f0c78bc8d2a75b451eb13e9 +size 1414118880 diff --git a/up07mbix/episode_rosbags/episode_14_2025-01-17-14-58-54.bag b/up07mbix/episode_rosbags/episode_14_2025-01-17-14-58-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..95991fc9257748d24bfee3e776ed171b6e598aa2 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_14_2025-01-17-14-58-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb861d1cca0d562a18f617d0c4c6bc9bb41a8b1c2241d5d25fc81feeac1991a +size 1412323386 diff --git a/up07mbix/episode_rosbags/episode_15_2025-01-17-14-59-46.bag b/up07mbix/episode_rosbags/episode_15_2025-01-17-14-59-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..f2d8d063c3b3fb4c0ce25127951942228ea3162b --- /dev/null +++ b/up07mbix/episode_rosbags/episode_15_2025-01-17-14-59-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4cefd45bf464cd0405d5791beb926260ce9a574c97de5885dd366ee47289db2 +size 1410881443 diff --git a/up07mbix/episode_rosbags/episode_16_2025-01-17-15-00-28.bag b/up07mbix/episode_rosbags/episode_16_2025-01-17-15-00-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..21954a36ee0be5c39411a8101cbd1c7242fc1121 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_16_2025-01-17-15-00-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0112a4b13cdeba8b2c41e7befe6823c2af42204d2af26fd442d88d5d976c6f3d +size 1410545085 diff --git a/up07mbix/episode_rosbags/episode_17_2025-01-17-15-01-18.bag b/up07mbix/episode_rosbags/episode_17_2025-01-17-15-01-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..c6e9e4236ccd2023eb3eb7e2c1cf0044324e97d7 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_17_2025-01-17-15-01-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0df17e65e0755b68508067ba1874f31d03284ce53835331e62536d793edbc3c +size 1440364025 diff --git a/up07mbix/episode_rosbags/episode_18_2025-01-17-15-02-05.bag b/up07mbix/episode_rosbags/episode_18_2025-01-17-15-02-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..97b44569707521223496be813ac5f96046991e95 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_18_2025-01-17-15-02-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9fc0b43bc93582322fc66ab3547efd43618950b738645392aa1775aad9c761a +size 1411615316 diff --git a/up07mbix/episode_rosbags/episode_19_2025-01-17-15-02-52.bag b/up07mbix/episode_rosbags/episode_19_2025-01-17-15-02-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..2c3a66fd1f18da493120176719f4dbade9c9aa35 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_19_2025-01-17-15-02-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691bb694ec5a9acf8717ef95fc6eef6462b684d599063bf23e025b83d5011519 +size 1410483815 diff --git a/up07mbix/episode_rosbags/episode_1_2025-01-17-14-48-58.bag b/up07mbix/episode_rosbags/episode_1_2025-01-17-14-48-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..43c101711a4b6cc1113f8794f1803565ceb03c44 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_1_2025-01-17-14-48-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6752a46bbdd077674e28a7cc633575408f9b8398fdf89d7484d88a23f0bf2b +size 1412972149 diff --git a/up07mbix/episode_rosbags/episode_2_2025-01-17-14-49-45.bag b/up07mbix/episode_rosbags/episode_2_2025-01-17-14-49-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..d09b5d63d2a0afc424ffab553b3c07252a588ef0 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_2_2025-01-17-14-49-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ae1d2dd63458e82f54039f79f35a8e3ac5b08a1a60bc85e9ad86c721e119f07 +size 1406043781 diff --git a/up07mbix/episode_rosbags/episode_3_2025-01-17-14-50-30.bag b/up07mbix/episode_rosbags/episode_3_2025-01-17-14-50-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..be8573818dc0b3d82467466a9e577cd868d9018d --- /dev/null +++ b/up07mbix/episode_rosbags/episode_3_2025-01-17-14-50-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5442ff03ade1c964831233257157f34a214c2e27678906735c8fdbf9df21b1a4 +size 1413049103 diff --git a/up07mbix/episode_rosbags/episode_4_2025-01-17-14-51-15.bag b/up07mbix/episode_rosbags/episode_4_2025-01-17-14-51-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..d0d45e33d9dc46b77926bcfc4130b46f77d66c74 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_4_2025-01-17-14-51-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2ce272ca16a6269bc915add323fc2c82bf8b38708c9790c7f262f3c2560dfa2 +size 1412574906 diff --git a/up07mbix/episode_rosbags/episode_5_2025-01-17-14-51-58.bag b/up07mbix/episode_rosbags/episode_5_2025-01-17-14-51-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..d3715ac2b5595ba5537c154db674373d32ae7aa9 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_5_2025-01-17-14-51-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fef04200c8af7152ff9df507c469e751f42894493c22804a14b8300711e647 +size 1414158278 diff --git a/up07mbix/episode_rosbags/episode_6_2025-01-17-14-52-43.bag b/up07mbix/episode_rosbags/episode_6_2025-01-17-14-52-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..7c17d2e404934d3a8078ab79a0c6c3c7fd63eb41 --- /dev/null +++ b/up07mbix/episode_rosbags/episode_6_2025-01-17-14-52-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8834992575a041ae1ebadd2609f814af3d907c26f115679d009fa9a3d5c262a7 +size 1409438216 diff --git a/up07mbix/episode_rosbags/episode_8_2025-01-17-14-54-08.bag b/up07mbix/episode_rosbags/episode_8_2025-01-17-14-54-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..c7fd18d14cf2718d9e3d473e7f19d238f0cefc7c --- /dev/null +++ b/up07mbix/episode_rosbags/episode_8_2025-01-17-14-54-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcf0113f417f798a4d51b4697cd05d9d019d305b6033c7db03332132ffa83f7f +size 1410632568 diff --git a/up07mbix/episode_rosbags/episode_9_2025-01-17-14-54-53.bag b/up07mbix/episode_rosbags/episode_9_2025-01-17-14-54-53.bag new file mode 100644 index 0000000000000000000000000000000000000000..af6bb7770de4553228b9b7a09470e9d00088c87e --- /dev/null +++ b/up07mbix/episode_rosbags/episode_9_2025-01-17-14-54-53.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99eaab1b63ea3539f0db390da860ebf4a45c72c97cc98502f87fcf8c7943b5b6 +size 1414457267 diff --git a/up07mbix/eval_robot.log b/up07mbix/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ebab9d6cf4cd4d99b534a872446079d71e209206 --- /dev/null +++ b/up07mbix/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-17 14:47:36,133][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-17 14:47:36,136][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/up07mbix/eval_video/0_eval.mp4 b/up07mbix/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1852f08214cb2bae8b506ea2a13c9899bb72bf91 --- /dev/null +++ b/up07mbix/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1819f930b352d4eea54cb805305cebbd6f2ce37f51bd15e133ca4ef3bf2aea7a +size 1144404 diff --git a/up07mbix/eval_video/10_eval.mp4 b/up07mbix/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3e8eaed0920414de73ed74d3458ecc26defa8a53 --- /dev/null +++ b/up07mbix/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:268ff9e2e53b41188aa0fa482c7bfb19f2fb2785f83d12074c59151944a43b98 +size 1293306 diff --git a/up07mbix/eval_video/11_eval.mp4 b/up07mbix/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29073e5e7bce28170e8a2f465bf0f68f1b1aaa07 --- /dev/null +++ b/up07mbix/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30dc16b787631229f3663c1d3202233d965a1bc38599211ed848ed4abdd882da +size 1370113 diff --git a/up07mbix/eval_video/12_eval.mp4 b/up07mbix/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49dcd23278da926be22027f6085df6ca394fab5c --- /dev/null +++ b/up07mbix/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79b7fe799da0fe5e29e5c659a71124a879c60c4ccf2e14eee9ca19e62f92aa6d +size 1367490 diff --git a/up07mbix/eval_video/13_eval.mp4 b/up07mbix/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2fa5463d6f1038a1424e5acb5fb4c829814c7e6f --- /dev/null +++ b/up07mbix/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72586beaede9059dffaad451ccb47b783d6a63ddc277e0544b3f874c0377af41 +size 1447624 diff --git a/up07mbix/eval_video/14_eval.mp4 b/up07mbix/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11de900f1ac6226943f45269d8aa5b7200ce78e2 --- /dev/null +++ b/up07mbix/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd6e56a3506bb346792fc8d7c11910474f4099601c968e0484ff1921e7565a1 +size 1458401 diff --git a/up07mbix/eval_video/15_eval.mp4 b/up07mbix/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2234484bf4d88d3cbc387b85752c2bc7435b9650 --- /dev/null +++ b/up07mbix/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e02939fb08b9aeb13e66076664fc0010c1e76536ca48fa048578a2b52b106ed +size 1167199 diff --git a/up07mbix/eval_video/16_eval.mp4 b/up07mbix/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..daf8169414718911495fa6aa619228d486f984c9 --- /dev/null +++ b/up07mbix/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d19b3abe759fefca682750d3714304bc8492a89c5fc8d582c7add5e46b26341d +size 1515039 diff --git a/up07mbix/eval_video/17_eval.mp4 b/up07mbix/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f3bcf587784003c3388ab3e96731095414ac4e9 --- /dev/null +++ b/up07mbix/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9039e216f093cde1d3f441956a295f944bc8db517e4fbe27227dde92cf249a72 +size 1188164 diff --git a/up07mbix/eval_video/18_eval.mp4 b/up07mbix/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b43711926c51765071f6568508bfd816b4d50c4c --- /dev/null +++ b/up07mbix/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d73f4283df1c3c229b048e1ca89694edabce51c187862a795a376e5126820abb +size 1381842 diff --git a/up07mbix/eval_video/19_eval.mp4 b/up07mbix/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..030668e5cda701b572c703e0f9e032e51a52a624 --- /dev/null +++ b/up07mbix/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1265d28d813382198c39f4d4d8adce2a6ae621c1dce45866f8e1475c1d56acb7 +size 1572862 diff --git a/up07mbix/eval_video/1_eval.mp4 b/up07mbix/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a617ad35f4c6414d508e3343aa76e2142a3d324c --- /dev/null +++ b/up07mbix/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:093e197b5aca9677627b2fe474f40045296961c6d67cd343459930ef0fdfea31 +size 1188450 diff --git a/up07mbix/eval_video/2_eval.mp4 b/up07mbix/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..888e03f8fdd51b3c4a9aff3032318887640174eb --- /dev/null +++ b/up07mbix/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6ae32e8cb4b17611f28e875305214f9d7b534d2c4eb2e243db0f4b59e6a53f +size 1146552 diff --git a/up07mbix/eval_video/3_eval.mp4 b/up07mbix/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64a1606f90f37ce6318185900af3604bdc46905a --- /dev/null +++ b/up07mbix/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccfd7c2808a5399343dba4263c583d12e3f79c291c8523c1ac43c6dbbce9a53b +size 1148831 diff --git a/up07mbix/eval_video/4_eval.mp4 b/up07mbix/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..81c8432fbb8654b07afe726de47b0dea58ce361d --- /dev/null +++ b/up07mbix/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916d91435e19178c30f6ad3ccb21d41fcb4f5584d80aea0864a22682c8d60b5f +size 1159225 diff --git a/up07mbix/eval_video/5_eval.mp4 b/up07mbix/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e8e659b8b61884f8bdfc686c18d9e09527d5f9b --- /dev/null +++ b/up07mbix/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbb012eb02e7137181fe4902ad630dfc552a4d8a10b02437e92852077a2c37e +size 1229823 diff --git a/up07mbix/eval_video/6_eval.mp4 b/up07mbix/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..611ba0f4018e50b04a71acdd2aa13de7572a000d --- /dev/null +++ b/up07mbix/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a92f6e4d8653d4439383f6af37f6414a03e15252e2ef97c40394e947687cc0e5 +size 1162123 diff --git a/up07mbix/eval_video/7_eval.mp4 b/up07mbix/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8eb30d0879c8e4351fcb99db7a612be956d6daa8 --- /dev/null +++ b/up07mbix/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baeae50ab773d27ae44b9d6fd2c8a77836db35a896dec3de71b24adc6c06a9bc +size 1153241 diff --git a/up07mbix/eval_video/8_eval.mp4 b/up07mbix/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73ab8000b5d4e2ff657872d07127563ba30f6f1b --- /dev/null +++ b/up07mbix/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6383c0f370fe43d88dd9487aa2c5e8fccf8ac1b03a10253552f375d060493ea +size 1317618 diff --git a/up07mbix/eval_video/9_eval.mp4 b/up07mbix/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..75b30f16ea3bb383a4a4bd2f658f280895d83a0e --- /dev/null +++ b/up07mbix/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a200f16787e6024f65815a1c43b940d619d7f92303cb6b423b1d4573f8a4c2f8 +size 1343379 diff --git a/up07mbix/tb/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 b/up07mbix/tb/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 new file mode 100644 index 0000000000000000000000000000000000000000..db85c0f1918aa732cdba4e2f223f069f83784df6 --- /dev/null +++ b/up07mbix/tb/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a133d9cf3ca98a83050bb6fcad1a18edc73bbc5f28222d113f0464fe4a6c58 +size 22784 diff --git a/up07mbix/wandb/debug-internal.log b/up07mbix/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7007748425f1f183ab7912bfb66b8364bf1a3d39 --- /dev/null +++ b/up07mbix/wandb/debug-internal.log @@ -0,0 +1,2141 @@ +2025-01-17 14:47:41,921 INFO StreamThr :2517552 [internal.py:wandb_internal():85] W&B internal server running at pid: 2517552, started at: 2025-01-17 14:47:41.921158 +2025-01-17 14:47:41,922 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status +2025-01-17 14:47:41,933 INFO WriterThread:2517552 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb +2025-01-17 14:47:41,934 DEBUG SenderThread:2517552 [sender.py:send():379] send: header +2025-01-17 14:47:41,942 DEBUG SenderThread:2517552 [sender.py:send():379] send: run +2025-01-17 14:47:42,165 INFO SenderThread:2517552 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files +2025-01-17 14:47:42,165 INFO SenderThread:2517552 [sender.py:_start_run_threads():1188] run started: up07mbix with start time 1737143261.921632 +2025-01-17 14:47:42,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 14:47:42,182 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: check_version +2025-01-17 14:47:42,265 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 14:47:42,270 DEBUG HandlerThread:2517552 [system_info.py:__init__():26] System info init +2025-01-17 14:47:42,270 DEBUG HandlerThread:2517552 [system_info.py:__init__():41] System info init done +2025-01-17 14:47:42,270 INFO HandlerThread:2517552 [system_monitor.py:start():194] Starting system monitor +2025-01-17 14:47:42,270 INFO SystemMonitor:2517552 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 14:47:42,270 INFO HandlerThread:2517552 [system_monitor.py:probe():214] Collecting system info +2025-01-17 14:47:42,271 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 14:47:42,271 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started disk monitoring +2025-01-17 14:47:42,272 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 14:47:42,272 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started memory monitoring +2025-01-17 14:47:42,273 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started network monitoring +2025-01-17 14:47:42,282 DEBUG HandlerThread:2517552 [system_info.py:probe():152] Probing system +2025-01-17 14:47:42,286 DEBUG HandlerThread:2517552 [system_info.py:_probe_git():137] Probing git +2025-01-17 14:47:42,292 DEBUG HandlerThread:2517552 [system_info.py:_probe_git():145] Probing git done +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_info.py:probe():200] Probing system done +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T19:47:42.282982', 'startedAt': '2025-01-17T19:47:41.909118', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4134.258, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 399.98992919921875}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 14:47:42,293 INFO HandlerThread:2517552 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 14:47:42,293 INFO HandlerThread:2517552 [system_monitor.py:probe():227] Publishing system info +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_info.py:_save_code():44] Saving code +2025-01-17 14:47:42,294 DEBUG HandlerThread:2517552 [system_info.py:_save_code():67] Saving code done +2025-01-17 14:47:42,294 DEBUG HandlerThread:2517552 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 14:47:42,390 DEBUG HandlerThread:2517552 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 14:47:42,390 INFO HandlerThread:2517552 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 14:47:42,393 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:47:42,393 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 14:47:42,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 14:47:42,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: python_packages +2025-01-17 14:47:42,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:47:42,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:47:42,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:42,655 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/5fdjf9gk-code/FISH/eval_robot.py +2025-01-17 14:47:42,664 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/9ted8sye-wandb-metadata.json +2025-01-17 14:47:42,669 INFO HandlerThread:2517552 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/tb" + save: true +} +control { + relay_id: "ac8462253926464da2d1680d5c59cfdf" +} +_info { + stream_id: "up07mbix" +} + +2025-01-17 14:47:42,669 DEBUG HandlerThread:2517552 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 14:47:42,688 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/aoalkeoe-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:47:42,689 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/kci0a5wf-diff.patch +2025-01-17 14:47:42,691 DEBUG SenderThread:2517552 [sender.py:send():379] send: telemetry +2025-01-17 14:47:42,737 DEBUG SenderThread:2517552 [sender.py:send():379] send: tbrecord +2025-01-17 14:47:42,738 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:47:42,738 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 with policy live +2025-01-17 14:47:42,919 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/85ejh0y0-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:47:43,166 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code +2025-01-17 14:47:43,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:44,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:45,166 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:45,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:46,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:47,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:47,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:48,084 DEBUG SenderThread:2517552 [sender.py:send():379] send: config +2025-01-17 14:47:48,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:49,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:49,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:50,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:51,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:51,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:52,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:52,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:53,168 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:53,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:54,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:55,169 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:55,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:56,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:57,169 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:57,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:47:57,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:47:57,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:57,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:58,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:59,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:00,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:01,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:02,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:03,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:03,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:04,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:05,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:06,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:07,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:08,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:08,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:09,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:10,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:11,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:12,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:12,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:12,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:13,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:13,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:13,736 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:14,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 14:48:14,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:15,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:15,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:16,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:17,175 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:17,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:18,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:19,628 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:19,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:20,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:21,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:22,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:23,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:24,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:25,473 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:25,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:26,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:27,178 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:27,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:27,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:27,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:28,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:29,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:30,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:30,776 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:31,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:32,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:33,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:34,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:35,180 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:35,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:36,317 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:36,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:37,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:38,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:39,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:40,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:41,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:41,983 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:42,172 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:48:42,172 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 with policy now +2025-01-17 14:48:42,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:48:42,174 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:48:42,174 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:48:42,174 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media +2025-01-17 14:48:42,273 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 14:48:42,274 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:48:42,469 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ybp3svcy-media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:42,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:42,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:42,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:42,745 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/lbxqg749-media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:43,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:43,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:44,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:45,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:46,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:47,184 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:47,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:47,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:48,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:49,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:50,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:51,185 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:51,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:52,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:53,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:53,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:54,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:55,186 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:48:55,444 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/a46a0ts1-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:48:55,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:56,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:57,187 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:57,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:57,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:57,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:58,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:59,188 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:59,383 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:59,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:00,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:01,188 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:01,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:02,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:03,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:04,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:04,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:05,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:06,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:07,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:08,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:09,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:09,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:10,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:11,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:12,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:49:12,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:12,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:12,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:13,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:14,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:14,751 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:15,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:16,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:17,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:18,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:19,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:20,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:20,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:21,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:22,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:23,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:24,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:25,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:25,862 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:26,270 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:49:26,270 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 with policy now +2025-01-17 14:49:26,273 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:49:26,273 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:49:26,274 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:49:26,274 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:49:26,525 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hg6wq3ey-media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:26,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:49:27,484 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/b3bqb9i9-media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:27,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:27,634 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:27,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:28,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:29,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:30,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:30,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:31,199 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:31,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:32,665 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:33,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:34,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:35,201 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:35,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:36,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:36,666 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:37,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:38,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:39,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:40,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:41,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:42,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:42,275 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:49:42,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:42,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:42,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:43,142 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:49:43,142 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:49:43,142 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:49:43,203 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:49:43,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:43,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:49:43,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:44,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:45,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:45,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:46,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:47,205 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:47,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:47,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:48,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:49,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:50,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:51,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:52,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:52,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:53,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:54,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:55,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:56,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:57,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:57,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:57,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:57,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:58,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:59,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:00,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:01,210 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:01,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:02,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:03,211 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:03,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:03,656 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:04,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:05,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:06,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:07,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:08,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:09,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:09,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:10,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:11,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:12,275 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:50:12,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:12,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:12,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:13,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:14,407 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:50:14,408 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 with policy now +2025-01-17 14:50:14,410 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:50:14,410 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:14,411 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:14,411 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:14,658 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:14,665 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/cnvywjy0-media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:50:15,428 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:15,501 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/kgs37oz6-media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,660 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:16,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:17,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:17,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:18,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:19,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:19,671 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:20,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:20,823 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:21,676 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:22,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:23,219 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:23,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:24,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:25,676 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:26,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:26,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:27,220 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:50:27,490 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/mkcw0g0c-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:50:27,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:27,633 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:27,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:27,938 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:27,938 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:27,938 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:28,221 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:28,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:29,221 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:29,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:30,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:31,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:32,139 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:32,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:33,222 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:33,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:34,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:35,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:36,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:37,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:37,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:38,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:39,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:40,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:41,225 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:41,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:42,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:50:42,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:42,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:42,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:42,726 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:43,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:44,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:45,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:46,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:47,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:48,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:48,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:49,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:50,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:51,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:52,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:53,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:54,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:54,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:55,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:56,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:57,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:57,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:57,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:58,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:59,137 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:50:59,137 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:50:59,138 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 with policy now +2025-01-17 14:50:59,138 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:59,138 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:59,138 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,233 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:50:59,390 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ma9sq65m-media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,665 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/8lbemu2o-media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:00,165 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:00,693 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:01,233 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:01,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:02,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:03,685 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:04,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:05,234 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:05,565 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:05,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:06,688 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:07,235 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:07,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:08,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:09,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:10,642 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:10,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:11,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:11,992 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:11,993 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:11,993 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:12,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:12,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:12,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:51:12,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:12,641 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:12,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:13,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:13,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:14,699 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:15,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:16,546 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:16,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:17,239 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:17,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:18,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:19,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:20,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:21,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:21,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:22,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:23,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:24,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:25,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:26,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:26,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:27,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:27,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:27,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:28,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:29,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:30,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:31,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:31,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:32,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:33,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:34,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:35,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:36,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:37,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:37,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:38,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:39,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:40,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:41,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:42,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:51:42,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:42,631 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:42,706 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:42,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:43,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:43,381 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:51:43,382 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 with policy now +2025-01-17 14:51:43,386 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:51:43,387 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:43,387 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:43,388 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:43,632 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ixvd3p6b-media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:43,708 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:51:44,551 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qmgc5aka-media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:44,708 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:45,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:45,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:46,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:47,712 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:48,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:48,717 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:49,250 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:49,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:50,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:51,251 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:51,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:52,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:53,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:54,164 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:54,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:54,512 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/r2k4us42-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:54,725 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:55,044 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:55,045 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:55,045 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:55,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:55,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:55,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:56,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:57,253 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:57,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:57,638 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:57,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:58,733 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:59,253 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:59,727 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:59,964 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:00,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:01,254 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:01,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:02,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:03,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:04,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:05,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:05,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:06,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:07,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:08,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:09,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:10,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:10,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:11,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:12,277 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:52:12,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:12,631 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:12,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:13,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:14,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:15,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:15,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:16,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:17,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:18,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:19,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:20,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:21,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:21,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:22,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:23,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:24,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:25,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:26,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:26,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:27,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:27,643 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:27,738 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:28,247 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:52:28,247 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 with policy now +2025-01-17 14:52:28,254 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:52:28,255 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:52:28,255 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:52:28,255 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:52:28,264 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:52:28,264 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:28,265 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:52:28,504 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qdc9zoja-media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:28,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:28,766 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/o5g4fsr7-media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:29,265 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:29,738 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:30,737 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:31,736 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:31,925 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:32,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:33,266 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:33,744 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:34,751 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:35,750 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:36,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:37,268 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:37,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:37,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:38,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:39,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:40,107 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:52:40,107 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:52:40,108 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:52:40,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:52:40,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:52:40,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:41,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:41,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:42,277 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:52:42,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:42,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:42,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:42,777 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:43,270 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:43,758 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:44,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:45,270 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:45,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:46,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:47,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:48,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:48,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:49,272 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:49,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:50,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:51,273 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:51,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:52,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:53,274 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:53,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:53,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:54,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:55,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:56,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:57,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:57,635 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:57,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:58,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:58,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:59,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:00,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:01,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:02,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:03,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:04,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:04,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:05,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:06,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:07,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:08,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:09,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:10,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:10,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:11,769 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:12,129 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:53:12,129 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 with policy now +2025-01-17 14:53:12,137 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:53:12,138 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:12,138 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:12,141 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:12,278 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:53:12,281 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:12,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:53:12,382 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/md47hjpp-media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:12,645 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:12,665 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/6upv9n67-media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,780 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:13,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:13,773 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:14,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:15,283 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:15,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:15,786 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:16,791 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:17,283 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:17,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:18,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:19,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:20,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:20,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:21,285 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:21,790 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:22,285 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:53:22,538 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/b3hid05a-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:53:22,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:23,161 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:23,161 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:23,161 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:23,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:23,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:23,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:24,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:25,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:25,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:26,195 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:26,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:27,287 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:27,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:27,639 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:27,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:28,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:29,288 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:29,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:30,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:31,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:31,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:32,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:33,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:34,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:35,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:36,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:37,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:37,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:38,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:39,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:40,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:41,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:42,278 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:53:42,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:42,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:42,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:42,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:43,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:44,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:45,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:46,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:47,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:48,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:48,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:49,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:50,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:51,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:52,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:53,822 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:54,197 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:54,587 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:53:54,587 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 with policy now +2025-01-17 14:53:54,590 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:53:54,590 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:54,590 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:54,590 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:54,810 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:54,847 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ls4gvw2l-media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:53:55,578 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/pjw07pfa-media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,816 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:56,827 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:57,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:58,839 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:59,299 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:59,317 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:59,842 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:00,845 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:01,838 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:02,847 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:03,301 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:03,836 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:04,357 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:04,844 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:05,213 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:05,213 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:05,213 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:05,846 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:06,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:07,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:07,845 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:08,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:09,303 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:09,429 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:09,857 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:10,865 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:11,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:12,279 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:54:12,649 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:12,649 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:12,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:13,304 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:13,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:14,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:14,915 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:15,305 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:15,852 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:16,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:17,306 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:17,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:18,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:19,307 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:19,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:20,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:20,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:21,308 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:21,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:22,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:23,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:24,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:25,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:26,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:26,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:27,649 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:27,649 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:27,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:28,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:29,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:30,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:31,726 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:31,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:32,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:33,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:34,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:35,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:36,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:37,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:37,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:38,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:39,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:40,873 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:41,194 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:54:41,194 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 with policy now +2025-01-17 14:54:41,202 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:54:41,202 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:41,202 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:41,203 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:41,315 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:41,315 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:41,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:54:41,459 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/71638wfj-media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,740 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/yv16vlac-media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:42,279 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:54:42,667 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:42,667 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:42,667 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:42,905 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:43,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:43,897 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:44,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:45,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:46,896 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:47,317 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:47,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:47,901 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:48,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:49,318 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:49,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:50,273 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:50,274 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:50,274 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:50,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:50,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:50,589 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/bkkmsaqw-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:50,900 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:51,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:51,905 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:52,903 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:53,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:53,901 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:54,904 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:55,321 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:55,916 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:56,921 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:57,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:57,668 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:57,922 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:58,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:58,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:59,322 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:59,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:00,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:01,323 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:01,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:02,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:03,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:04,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:04,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:05,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:06,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:07,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:08,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:09,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:09,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:10,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:11,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:12,280 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:55:12,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:12,650 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:12,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:13,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:14,731 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:14,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:15,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:16,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:17,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:18,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:19,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:19,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:20,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:21,331 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:21,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:22,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:23,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:24,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:25,520 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:25,934 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:26,898 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:55:26,898 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 with policy now +2025-01-17 14:55:26,912 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:55:26,913 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:55:26,913 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:55:26,913 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:55:26,945 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:27,179 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/zay1i2n7-media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:55:27,614 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/a8q9styt-media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:27,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:27,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:28,936 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:29,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:29,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:30,599 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:30,949 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:31,335 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:31,971 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:32,934 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:33,932 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:34,950 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:35,337 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:35,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:35,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:36,950 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:37,965 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:38,341 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:55:38,341 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:55:38,342 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:55:38,342 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:55:38,968 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:39,343 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:55:39,343 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:39,952 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:40,953 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:41,025 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:41,344 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:41,976 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:42,280 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:55:42,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:42,662 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:42,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:43,344 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:43,954 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:44,962 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:45,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:46,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:47,005 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:47,346 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:47,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:48,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:49,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:50,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:51,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:52,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:52,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:53,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:54,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:55,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:56,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:57,810 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:57,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:58,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:59,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:00,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:01,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:02,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:03,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:03,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:04,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:05,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:06,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:07,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:08,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:08,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:09,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:10,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:11,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:12,281 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:56:12,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:12,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:13,004 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:13,356 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:13,966 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:13,988 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:14,061 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:56:14,061 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 with policy now +2025-01-17 14:56:14,064 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:56:14,064 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:56:14,064 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:56:14,065 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:56:14,336 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ct8nouz2-media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:56:14,632 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ucmgw36v-media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,970 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:15,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:15,976 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:16,977 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:17,974 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:18,997 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:19,359 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:19,773 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:19,984 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:20,969 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:21,361 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:21,983 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:22,988 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:23,361 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:24,021 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:25,012 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:25,182 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:25,362 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:56:25,396 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:56:25,397 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:56:25,397 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:56:25,644 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g762glol-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:56:26,012 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:26,363 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:56:27,022 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:27,363 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:27,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:27,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:28,011 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:29,016 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:30,013 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:30,428 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:31,048 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:31,365 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:32,043 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:33,051 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:34,051 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:35,056 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:35,483 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:36,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:37,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:37,367 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:38,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:39,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:40,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:40,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:41,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:42,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:42,289 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:56:42,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:42,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:43,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:44,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:45,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:45,370 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:46,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:46,545 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:47,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:48,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:49,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:50,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:51,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:51,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:52,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:53,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:54,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:55,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:56,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:56,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:57,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:58,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:59,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:00,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:01,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:01,725 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:02,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:03,068 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:03,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:03,754 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:57:03,754 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 with policy now +2025-01-17 14:57:03,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:57:03,765 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:03,765 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:03,766 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:04,013 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/yw7m3f6j-media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:04,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:04,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:04,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:04,379 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:57:04,674 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hssrhkv3-media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:05,056 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:05,379 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:06,060 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:06,847 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:07,094 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:08,064 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:09,076 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:09,380 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:10,074 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:11,076 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:11,381 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:12,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:12,281 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:57:12,282 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:12,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:12,678 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:13,098 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:13,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:57:13,451 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:13,451 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:13,453 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:14,095 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:14,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:14,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:57:15,108 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:15,383 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:16,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:17,120 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:17,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:18,125 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:19,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:19,384 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:20,136 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:21,140 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:22,128 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:22,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:23,132 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:24,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:25,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:25,386 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:26,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:27,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:27,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:27,653 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:27,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:28,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:29,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:30,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:31,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:32,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:33,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:33,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:34,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:35,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:36,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:37,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:37,391 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:38,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:39,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:39,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:40,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:41,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:42,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:42,282 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:57:42,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:42,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:43,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:44,116 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:44,758 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:45,116 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:46,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:47,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:48,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:49,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:49,396 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:50,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:50,313 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:51,150 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:51,397 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:51,487 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:57:51,487 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 with policy now +2025-01-17 14:57:51,493 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:57:51,493 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:51,493 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:51,494 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:51,748 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hqx05krv-media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:52,133 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:57:52,707 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/u4scvtze-media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:53,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:53,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:54,152 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:55,193 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:56,150 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:56,193 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:57,159 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:57,400 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:57,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:57,684 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:58,146 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:59,154 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:59,401 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:00,145 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:01,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:01,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:02,149 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:02,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:02,514 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:02,514 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:02,514 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:03,164 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:03,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:03,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:04,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:05,148 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:05,403 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:06,161 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:07,152 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:07,205 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:07,404 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:08,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:09,176 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:10,169 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:11,169 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:12,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:12,282 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:58:12,283 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:12,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:12,672 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:13,179 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:14,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:15,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:15,407 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:16,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:17,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:17,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:18,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:19,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:20,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:21,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:22,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:22,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:23,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:24,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:25,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:26,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:27,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:27,665 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:27,666 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:27,813 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:28,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:29,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:30,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:31,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:32,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:33,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:33,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:34,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:35,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:36,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:37,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:38,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:38,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:39,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:40,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:41,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:41,418 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:42,179 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:42,271 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:58:42,271 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 with policy now +2025-01-17 14:58:42,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:58:42,285 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:58:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:42,285 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:42,286 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:42,418 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:42,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:42,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:58:42,540 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/x4gcj7og-media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:42,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:42,672 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:42,812 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/alyp4d5r-media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:43,204 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:43,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:43,783 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:44,212 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:45,203 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:45,420 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:46,242 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:47,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:47,420 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:48,220 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:49,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:49,240 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:50,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:50,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:50,576 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:50,576 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:50,581 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:50,695 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/jbot3eff-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:51,233 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:52,269 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:53,263 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:53,423 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:54,236 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:54,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:55,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:55,424 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:56,261 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:57,238 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:57,686 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:57,686 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:58,235 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:59,262 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:59,844 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:00,288 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:01,258 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:02,265 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:03,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:03,427 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:04,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:05,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:05,309 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:05,428 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:06,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:07,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:08,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:09,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:10,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:10,752 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:11,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:12,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:12,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:59:12,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:12,673 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:13,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:14,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:15,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:15,775 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:16,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:17,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:18,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:19,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:20,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:21,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:21,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:22,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:23,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:24,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:25,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:26,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:27,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:27,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:27,674 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:27,795 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:28,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:29,249 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:30,297 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:31,274 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:31,716 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:59:31,716 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 with policy now +2025-01-17 14:59:31,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:59:31,721 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:59:31,721 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:59:31,721 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:59:31,998 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/1ah9u18q-media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,281 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:32,440 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:59:32,440 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:32,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:59:32,753 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/9n8n3cul-media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,807 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:33,287 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:33,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:34,290 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:34,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:35,311 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:36,295 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:37,298 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:38,296 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:38,443 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:38,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:39,300 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:40,297 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:40,444 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:41,313 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:42,284 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:59:42,329 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:42,444 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:59:42,641 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:59:42,641 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:59:42,642 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:59:42,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:42,697 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:43,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:43,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:59:43,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:59:44,322 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:44,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:44,477 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:45,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:46,360 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:46,446 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:47,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:48,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:48,447 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:49,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:49,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:50,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:50,448 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:51,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:52,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:53,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:54,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:55,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:55,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:56,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:57,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:57,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:57,679 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:58,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:59,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:00,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:00,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:01,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:02,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:03,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:04,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:05,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:06,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:06,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:07,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:08,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:09,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:10,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:11,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:11,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:12,284 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:00:12,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:12,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:12,680 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:13,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:14,345 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:15,348 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:15,621 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:00:15,621 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 with policy now +2025-01-17 15:00:15,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:00:15,644 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:00:15,644 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:00:15,644 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:00:15,886 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/553ul7rh-media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:16,345 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:00:16,753 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/51414zxw-media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:17,347 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:17,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:17,734 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:18,350 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:18,460 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:19,350 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:20,353 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:21,376 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:22,363 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:22,461 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:22,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:23,392 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:23,688 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:00:23,688 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:00:23,693 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:00:24,391 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:24,462 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:00:24,462 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:24,463 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:00:25,425 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:26,400 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:26,463 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:27,416 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:27,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:27,694 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:27,846 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:28,423 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:29,421 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:30,419 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:30,465 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:31,421 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:32,436 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:33,463 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:33,814 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:34,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:35,466 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:36,469 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:37,486 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:38,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:38,982 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:39,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:40,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:40,469 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:41,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:00:42,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:42,684 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:42,684 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:43,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:44,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:44,964 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:45,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:46,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:47,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:48,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:49,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:50,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:50,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:51,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:52,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:53,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:54,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:55,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:55,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:56,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:57,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:57,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:57,692 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:58,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:59,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:00,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:00,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:01,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:02,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:03,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:04,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:05,480 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:06,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:06,213 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:01:06,213 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 with policy now +2025-01-17 15:01:06,235 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:01:06,236 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:06,236 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:06,236 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:06,473 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:06,481 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:01:06,483 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/5skhsm87-media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:06,756 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/mctt4p8j-media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:07,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:07,493 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:08,481 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:08,501 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:09,583 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:10,550 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:11,553 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:11,981 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:12,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:01:12,483 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:12,546 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:12,734 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:12,735 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:13,530 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:14,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:01:14,539 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:14,755 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:14,755 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:14,755 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:14,780 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g83t1x47-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:01:15,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:15,577 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:16,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:16,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:17,245 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:17,606 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:18,485 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:18,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:19,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:20,486 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:20,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:21,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:22,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:22,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:23,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:24,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:25,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:26,489 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:26,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:27,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:27,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:27,691 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:27,812 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:28,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:29,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:30,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:31,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:32,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:32,824 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:33,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:34,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:35,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:36,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:37,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:37,824 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:38,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:39,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:40,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:41,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:01:42,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:42,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:42,692 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:42,838 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:43,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:44,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:45,576 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:46,498 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:46,583 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:47,580 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:47,767 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:01:47,767 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 with policy now +2025-01-17 15:01:47,793 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:01:47,793 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:47,793 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:47,793 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:47,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:48,035 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/lqhk7obw-media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:01:48,624 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:48,794 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/3avpp5qc-media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:49,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:49,612 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:50,500 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:50,595 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:51,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:52,623 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:52,897 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:53,618 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:54,501 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:54,599 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:55,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:56,502 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:56,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:57,615 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:57,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:57,756 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:57,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:58,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:59,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:00,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:00,811 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:00,811 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:00,812 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:01,504 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:01,505 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:02:01,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:02,505 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:02,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:03,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:03,883 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:04,506 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:04,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:05,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:06,507 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:06,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:07,671 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:08,508 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:08,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:09,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:09,873 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:10,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:11,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:12,286 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:02:12,509 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:12,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:12,712 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:12,713 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:13,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:14,510 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:14,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:15,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:15,863 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:16,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:17,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:18,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:19,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:20,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:21,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:21,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:22,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:23,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:24,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:25,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:26,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:26,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:27,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:27,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:27,713 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:28,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:29,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:30,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:31,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:31,861 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:32,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:33,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:34,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:35,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:36,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:37,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:37,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:38,657 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:39,426 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:02:39,426 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 with policy now +2025-01-17 15:02:39,447 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:02:39,448 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:39,448 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:39,448 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:39,521 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:39,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:39,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:02:39,659 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:39,699 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qxfe0zj5-media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:39,976 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/sxl2cva2-media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:40,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:40,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:41,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:41,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:42,286 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:02:42,523 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:42,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:42,772 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:42,773 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:42,912 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:43,711 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:44,524 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:44,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:45,729 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:46,525 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:46,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:47,685 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:47,868 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:47,869 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:47,869 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:47,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:02:48,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:49,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:50,527 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:50,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:51,714 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:52,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:53,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:53,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:54,528 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:54,730 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:55,764 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:56,741 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:57,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:57,764 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:57,765 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:58,742 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:59,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:59,904 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:00,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:01,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:02,793 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:03,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:04,533 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:04,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:05,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:05,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:06,535 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:06,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:07,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:08,536 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:08,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:09,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:10,537 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:10,745 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:10,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:11,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:12,287 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:03:12,538 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:12,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:12,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:12,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:13,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:14,539 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:14,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:15,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:15,985 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:16,540 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:16,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:17,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:18,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:19,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:20,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:21,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:21,891 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:22,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:23,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:24,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:25,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:26,544 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:26,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:27,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:27,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:27,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:27,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:28,545 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:28,757 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:29,757 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:30,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:31,602 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:31,602 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 with policy now +2025-01-17 15:03:31,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:31,640 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:31,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:31,641 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:31,780 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:31,872 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/x46k3m8c-media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:32,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:03:32,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:32,871 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/jl6hd4vq-media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:33,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:33,791 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:33,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:34,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:34,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:35,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:36,839 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:37,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:38,550 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:38,831 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:38,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:39,550 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:39,834 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:39,932 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:39,932 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:39,933 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:40,831 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:40,921 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 15:03:40,921 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 15:03:41,139 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:41,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table +2025-01-17 15:03:41,558 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 15:03:41,570 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:41,635 INFO wandb-upload_2:2517552 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf7i9yzmj +2025-01-17 15:03:41,793 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/thkplcdy-media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:41,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:41,855 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qtky6ri1-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:42,076 INFO SenderThread:2517552 [sender.py:send_request_log_artifact():1525] logged artifact run-up07mbix-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEzMjcwMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU2Mjc2NA==', 'latestArtifact': None}} +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:42,076 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json with policy now +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,077 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,077 DEBUG SenderThread:2517552 [sender.py:send():379] send: telemetry +2025-01-17 15:03:42,077 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,081 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,081 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,081 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,082 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,082 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,082 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,083 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,083 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,083 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,083 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,084 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,084 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,084 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,085 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,085 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,085 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,085 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,086 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,086 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,086 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,087 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,087 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 15:03:42,343 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ulxn4kk7-media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:42,473 INFO wandb-upload_3:2517552 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppfo4cq0c +2025-01-17 15:03:42,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval +2025-01-17 15:03:42,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:42,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:42,904 INFO SenderThread:2517552 [sender.py:send_request_log_artifact():1525] logged artifact run-up07mbix-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEzMjcyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU2Mjc2Nw==', 'latestArtifact': None}} +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:42,904 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json with policy now +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,905 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,905 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:03:42,905 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:43,158 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/6l91m2mj-media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:43,428 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g6lnbs6v-media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:43,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:43,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:43,972 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:44,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:44,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:45,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:46,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:47,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:48,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:49,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:49,894 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:50,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:51,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:52,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:53,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:54,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:54,894 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:55,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:56,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:57,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:57,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:57,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:58,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:59,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:59,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:00,561 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 15:04:00,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:01,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:02,405 DEBUG SenderThread:2517552 [sender.py:send():379] send: exit +2025-01-17 15:04:02,405 INFO SenderThread:2517552 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_exit():588] handling runtime: 980 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_exit():594] send defer +2025-01-17 15:04:02,406 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,406 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 15:04:02,406 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 1 +2025-01-17 15:04:02,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 15:04:02,407 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,407 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 15:04:02,407 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 2 +2025-01-17 15:04:02,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 15:04:02,407 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 15:04:02,408 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 15:04:02,408 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 15:04:02,458 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 15:04:02,459 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 15:04:02,459 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined network monitor +2025-01-17 15:04:02,459 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,459 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 15:04:02,459 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 3 +2025-01-17 15:04:02,459 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:04:02,460 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,460 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 15:04:02,460 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,460 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 15:04:02,460 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 4 +2025-01-17 15:04:02,460 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,460 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 15:04:02,562 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:04,564 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,942 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 15:04:08,942 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 5 +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:04:08,943 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:04:08,943 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,943 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 15:04:08,944 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:08,944 DEBUG SenderThread:2517552 [sender.py:send():379] send: summary +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:08,948 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 6 +2025-01-17 15:04:08,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,948 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 15:04:08,948 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 7 +2025-01-17 15:04:08,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:08,949 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,949 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 15:04:08,949 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,949 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 15:04:09,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:09,566 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:11,093 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 8 +2025-01-17 15:04:11,093 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:11,094 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,094 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 15:04:11,094 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,094 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 15:04:11,094 INFO SenderThread:2517552 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 15:04:11,097 INFO SenderThread:2517552 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 15:04:11,099 INFO SenderThread:2517552 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 15:04:11,113 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 9 +2025-01-17 15:04:11,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,114 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 15:04:11,114 DEBUG SenderThread:2517552 [sender.py:send():379] send: artifact +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'versionIndex': 442}}} +2025-01-17 15:04:11,314 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 15:04:11,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:11,567 INFO SenderThread:2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:11,567 INFO SenderThread:2517552 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml config.yaml +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json wandb-summary.json +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt requirements.txt +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json wandb-metadata.json +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log output.log +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch diff.patch +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 15:04:11,574 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 10 +2025-01-17 15:04:11,576 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:11,576 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,577 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 15:04:11,577 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,577 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 15:04:11,577 INFO SenderThread:2517552 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:04:11,763 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:11,792 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt +2025-01-17 15:04:11,805 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 15:04:11,814 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:12,015 INFO Thread-11 (_thread_body):2517552 [sender.py:transition_state():614] send defer: 11 +2025-01-17 15:04:12,015 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,015 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 15:04:12,015 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,015 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 15:04:12,015 INFO SenderThread:2517552 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 12 +2025-01-17 15:04:12,019 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,019 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 15:04:12,019 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [file_stream.py:finish():601] file stream finish called +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [file_stream.py:finish():605] file stream finish is done +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 13 +2025-01-17 15:04:12,217 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,217 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 15:04:12,217 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 14 +2025-01-17 15:04:12,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,218 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send():379] send: final +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send():379] send: footer +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,218 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 15:04:12,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:12,219 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:12,219 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: server_info +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 15:04:12,221 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 15:04:12,222 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:12,282 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:12,282 INFO MainThread:2517552 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 15:04:12,283 INFO MainThread:2517552 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 15:04:12,283 INFO MainThread:2517552 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 15:04:12,283 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 15:04:12,283 INFO HandlerThread:2517552 [handler.py:finish():882] shutting down handler +2025-01-17 15:04:13,220 INFO WriterThread:2517552 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [sender.py:finish():1615] shutting down sender +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [file_pusher.py:join():175] waiting for file pusher diff --git a/up07mbix/wandb/debug.log b/up07mbix/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..679e797b9970446285c36af14b01b59cafb466a3 --- /dev/null +++ b/up07mbix/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Configure stats pid to 2517388 +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/settings +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug.log +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug-internal.log +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():569] calling init triggers +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix'} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():619] starting backend +2025-01-17 14:47:41,916 INFO MainThread:2517388 [wandb_init.py:init():623] setting up manager +2025-01-17 14:47:41,920 INFO MainThread:2517388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 14:47:41,921 INFO MainThread:2517388 [wandb_init.py:init():631] backend started and connected +2025-01-17 14:47:41,933 INFO MainThread:2517388 [wandb_init.py:init():720] updated telemetry +2025-01-17 14:47:41,939 INFO MainThread:2517388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 14:47:42,181 INFO MainThread:2517388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 14:47:42,220 INFO MainThread:2517388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 14:47:42,220 INFO MainThread:2517388 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 14:47:42,628 INFO MainThread:2517388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 14:47:42,628 INFO MainThread:2517388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/tb, True +2025-01-17 14:47:48,084 INFO MainThread:2517388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 15:04:13,286 WARNING MsgRouterThr:2517388 [router.py:message_loop():77] message_loop has been closed diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py b/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml b/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75c4849ba83de31d1bcb3a71ff9578c29f6baf7e --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_0' +wandb_notes: + desc: null + value: 3456_0_req_3921_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737143261 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch b/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..9505998782078316d897b259427f75a7fd990633 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch @@ -0,0 +1,78 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,11 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9097a7c97ab11c8d7301524be72267ffb2892074 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.8014023005359567, 1, 0.8370807170193759, 1, 0.9347738473737921, 1, 0.9148586093549707, 1]]} \ No newline at end of file diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a292d3d4a800dada63c5c7683a11e53e2f97aa3a --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[66, 80, 0.825, 0.7136867939998497, 0.9083331971543944, 0.7333697080145443, 0.8928065522719597, 0.7536826841025817, 0.8825706876049224, 0.7583227343340642, 0.875470220967539]]} \ No newline at end of file diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1852f08214cb2bae8b506ea2a13c9899bb72bf91 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1819f930b352d4eea54cb805305cebbd6f2ce37f51bd15e133ca4ef3bf2aea7a +size 1144404 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3e8eaed0920414de73ed74d3458ecc26defa8a53 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:268ff9e2e53b41188aa0fa482c7bfb19f2fb2785f83d12074c59151944a43b98 +size 1293306 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29073e5e7bce28170e8a2f465bf0f68f1b1aaa07 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30dc16b787631229f3663c1d3202233d965a1bc38599211ed848ed4abdd882da +size 1370113 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49dcd23278da926be22027f6085df6ca394fab5c --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79b7fe799da0fe5e29e5c659a71124a879c60c4ccf2e14eee9ca19e62f92aa6d +size 1367490 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2fa5463d6f1038a1424e5acb5fb4c829814c7e6f --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72586beaede9059dffaad451ccb47b783d6a63ddc277e0544b3f874c0377af41 +size 1447624 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11de900f1ac6226943f45269d8aa5b7200ce78e2 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd6e56a3506bb346792fc8d7c11910474f4099601c968e0484ff1921e7565a1 +size 1458401 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2234484bf4d88d3cbc387b85752c2bc7435b9650 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e02939fb08b9aeb13e66076664fc0010c1e76536ca48fa048578a2b52b106ed +size 1167199 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..daf8169414718911495fa6aa619228d486f984c9 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d19b3abe759fefca682750d3714304bc8492a89c5fc8d582c7add5e46b26341d +size 1515039 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4f3bcf587784003c3388ab3e96731095414ac4e9 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9039e216f093cde1d3f441956a295f944bc8db517e4fbe27227dde92cf249a72 +size 1188164 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b43711926c51765071f6568508bfd816b4d50c4c --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d73f4283df1c3c229b048e1ca89694edabce51c187862a795a376e5126820abb +size 1381842 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..030668e5cda701b572c703e0f9e032e51a52a624 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1265d28d813382198c39f4d4d8adce2a6ae621c1dce45866f8e1475c1d56acb7 +size 1572862 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a617ad35f4c6414d508e3343aa76e2142a3d324c --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:093e197b5aca9677627b2fe474f40045296961c6d67cd343459930ef0fdfea31 +size 1188450 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..888e03f8fdd51b3c4a9aff3032318887640174eb --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6ae32e8cb4b17611f28e875305214f9d7b534d2c4eb2e243db0f4b59e6a53f +size 1146552 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..64a1606f90f37ce6318185900af3604bdc46905a --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccfd7c2808a5399343dba4263c583d12e3f79c291c8523c1ac43c6dbbce9a53b +size 1148831 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..81c8432fbb8654b07afe726de47b0dea58ce361d --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916d91435e19178c30f6ad3ccb21d41fcb4f5584d80aea0864a22682c8d60b5f +size 1159225 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e8e659b8b61884f8bdfc686c18d9e09527d5f9b --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbb012eb02e7137181fe4902ad630dfc552a4d8a10b02437e92852077a2c37e +size 1229823 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..611ba0f4018e50b04a71acdd2aa13de7572a000d --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a92f6e4d8653d4439383f6af37f6414a03e15252e2ef97c40394e947687cc0e5 +size 1162123 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8eb30d0879c8e4351fcb99db7a612be956d6daa8 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baeae50ab773d27ae44b9d6fd2c8a77836db35a896dec3de71b24adc6c06a9bc +size 1153241 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73ab8000b5d4e2ff657872d07127563ba30f6f1b --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6383c0f370fe43d88dd9487aa2c5e8fccf8ac1b03a10253552f375d060493ea +size 1317618 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..75b30f16ea3bb383a4a4bd2f658f280895d83a0e --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a200f16787e6024f65815a1c43b940d619d7f92303cb6b423b1d4573f8a4c2f8 +size 1343379 diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log b/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2d9af71fc472927e4ed276b7e6c4a051afbc6801 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log @@ -0,0 +1,1115 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737143268.084790]: resetting environment +[INFO] [1737143268.088543]: cleared current plan +[INFO] [1737143268.089517]: moving to home +[INFO] [1737143269.092279]: reached home +[INFO] [1737143269.093609]: reset action history +[INFO] [1737143271.801919]: environment reset +Starting episode 0 +[INFO] [1737143271.803783]: resetting environment +[INFO] [1737143271.808263]: cleared current plan +[INFO] [1737143271.809389]: moving to home +[INFO] [1737143272.811779]: reached home +[INFO] [1737143272.813245]: reset action history +[INFO] [1737143275.522389]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737143290.500278]: resetting environment +[INFO] [1737143290.503727]: cleared current plan +[INFO] [1737143290.504531]: moving to home +[INFO] [1737143291.506243]: reached home +[INFO] [1737143291.507412]: reset action history +[INFO] [1737143294.264575]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_0_2025-01-17-14-48-14.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:43483/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2517667] +process[print_text-2]: started with pid [2517691] +process[rosbag_record-3]: started with pid [2517692] +started bagging! +For topic gripper_width: timestamp difference is -109857847 for nearest: 1737143305277280043 - target: 1737143305387137890 at idx 153 +For topic gripper_width: timestamp difference is -109798490 for nearest: 1737143313077234686 - target: 1737143313187033176 at idx 249 +[INFO] [1737143321.247970]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737143321.983214]: Stored episode 1. +[INFO] [1737143321.983476]: Saving video... +[INFO] [1737143322.155532]: Video saved! +Starting episode 1 +[INFO] [1737143322.178349]: resetting environment +[INFO] [1737143322.182905]: cleared current plan +[INFO] [1737143322.183325]: moving to home +[INFO] [1737143325.785222]: reached home +[INFO] [1737143325.785697]: reset action history +[INFO] [1737143328.495191]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143334.424856]: resetting environment +[INFO] [1737143334.427599]: cleared current plan +[INFO] [1737143334.427757]: moving to home +[INFO] [1737143335.429191]: reached home +[INFO] [1737143335.429605]: reset action history +[INFO] [1737143338.137150]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_1_2025-01-17-14-48-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:34199/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2517814] +process[print_text-5]: started with pid [2517815] +process[rosbag_record-6]: started with pid [2517839] +started bagging! +[INFO] [1737143365.099108]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737143365.861333]: Stored episode 2. +[INFO] [1737143365.862056]: Saving video... +[INFO] [1737143366.243221]: Video saved! +Starting episode 2 +[INFO] [1737143366.273808]: resetting environment +[INFO] [1737143366.289668]: cleared current plan +[INFO] [1737143366.289943]: moving to home +[INFO] [1737143369.893702]: reached home +[INFO] [1737143369.898883]: reset action history +[INFO] [1737143372.622925]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143382.270033]: resetting environment +[INFO] [1737143382.280713]: cleared current plan +[INFO] [1737143382.281536]: moving to home +[INFO] [1737143383.283607]: reached home +[INFO] [1737143383.284231]: reset action history +[INFO] [1737143385.991436]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_2_2025-01-17-14-49-45.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:45605/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2517956] +process[print_text-8]: started with pid [2517957] +process[rosbag_record-9]: started with pid [2517958] +started bagging! +For topic gripper_width: timestamp difference is -104991099 for nearest: 1737143400177280763 - target: 1737143400282271862 at idx 196 +For topic gripper_width: timestamp difference is -100874460 for nearest: 1737143402177319967 - target: 1737143402278194427 at idx 226 +[INFO] [1737143412.935295]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737143413.730489]: Stored episode 3. +[INFO] [1737143413.735879]: Saving video... +[INFO] [1737143414.376334]: Video saved! +[WARN] [1737143414.299205]: Plan exhausted +[WARN] [1737143414.338611]: Plan exhausted +[WARN] [1737143414.379662]: Plan exhausted +[WARN] [1737143414.419304]: Plan exhausted +Starting episode 3 +[INFO] [1737143414.413833]: resetting environment +[INFO] [1737143414.427277]: cleared current plan +[INFO] [1737143414.427730]: moving to home +[INFO] [1737143418.030766]: reached home +[INFO] [1737143418.040890]: reset action history +[INFO] [1737143420.820031]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143427.083722]: resetting environment +[INFO] [1737143427.089830]: cleared current plan +[INFO] [1737143427.090051]: moving to home +[INFO] [1737143428.093243]: reached home +[INFO] [1737143428.098852]: reset action history +[INFO] [1737143430.878851]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_3_2025-01-17-14-50-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:34127/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2518104] +process[print_text-11]: started with pid [2518105] +process[rosbag_record-12]: started with pid [2518129] +started bagging! +For topic gripper_width: timestamp difference is -101743602 for nearest: 1737143439077311612 - target: 1737143439179055214 at idx 113 +[INFO] [1737143457.846016]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit[pub_text-10] killing on exit +[INFO] [1737143458.638453]: Stored episode 4. +[INFO] [1737143458.648072]: Saving video... +[INFO] [1737143459.093663]: Video saved! +Starting episode 4 +[INFO] [1737143459.129731]: resetting environment +[INFO] [1737143459.158889]: cleared current plan +[INFO] [1737143459.164455]: moving to home +[INFO] [1737143462.774381]: reached home +[INFO] [1737143462.779711]: reset action history +[INFO] [1737143465.558516]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143471.321851]: resetting environment +[INFO] [1737143471.338036]: cleared current plan +[INFO] [1737143471.341300]: moving to home +[INFO] [1737143472.352711]: reached home +[INFO] [1737143472.362074]: reset action history +[INFO] [1737143475.146130]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_4_2025-01-17-14-51-15.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:40827/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2518259] +process[print_text-14]: started with pid [2518260] +process[rosbag_record-15]: started with pid [2518261] +started bagging! +[INFO] [1737143502.252750]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737143503.070726]: Stored episode 5. +[INFO] [1737143503.072343]: Saving video... +[INFO] [1737143503.335600]: Video saved! +Starting episode 5 +[INFO] [1737143503.390572]: resetting environment +[INFO] [1737143503.421299]: cleared current plan +[INFO] [1737143503.422974]: moving to home +[INFO] [1737143507.027663]: reached home +[INFO] [1737143507.048058]: reset action history +[INFO] [1737143509.907465]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143514.202691]: resetting environment +[INFO] [1737143514.231632]: cleared current plan +[INFO] [1737143514.245265]: moving to home +[INFO] [1737143515.253654]: reached home +[INFO] [1737143515.261936]: reset action history +[INFO] [1737143518.052808]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_5_2025-01-17-14-51-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:45751/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2518403] +process[print_text-17]: started with pid [2518404] +process[rosbag_record-18]: started with pid [2518405] +started bagging! +[INFO] [1737143546.977538]: Storing episode... +[pub_text-16] killing on exit +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[INFO] [1737143547.750033]: Stored episode 6. +[INFO] [1737143547.753581]: Saving video... +[INFO] [1737143548.193529]: Video saved! +Starting episode 6 +[INFO] [1737143548.260392]: resetting environment +[INFO] [1737143548.289399]: cleared current plan +[INFO] [1737143548.305911]: moving to home +[INFO] [1737143551.917779]: reached home +[INFO] [1737143551.931110]: reset action history +[INFO] [1737143554.791203]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143559.513331]: resetting environment +[INFO] [1737143559.539080]: cleared current plan +[INFO] [1737143559.543762]: moving to home +[INFO] [1737143560.560376]: reached home +[INFO] [1737143560.573504]: reset action history +[INFO] [1737143563.425366]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_6_2025-01-17-14-52-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:42693/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2518549] +process[print_text-20]: started with pid [2518550] +process[rosbag_record-21]: started with pid [2518551] +started bagging! +For topic gripper_width: timestamp difference is 102272495 for nearest: 1737143568077313646 - target: 1737143567975041151 at idx 67 +For topic gripper_width: timestamp difference is 102391025 for nearest: 1737143568277279397 - target: 1737143568174888372 at idx 68 +For topic gripper_width: timestamp difference is 103037825 for nearest: 1737143569077301970 - target: 1737143568974264145 at idx 83 +For topic gripper_width: timestamp difference is 105688090 for nearest: 1737143572477280373 - target: 1737143572371592283 at idx 126 +[INFO] [1737143590.667457]: Storing episode... +[print_text-20] killing on exit +[rosbag_record-21] killing on exit +[pub_text-19] killing on exit +[INFO] [1737143591.478134]: Stored episode 7. +[INFO] [1737143591.494561]: Saving video... +[INFO] [1737143592.079415]: Video saved! +Starting episode 7 +[INFO] [1737143592.141655]: resetting environment +[INFO] [1737143592.162784]: cleared current plan +[INFO] [1737143592.173723]: moving to home +[WARN] [1737143592.009199]: Plan exhausted +[WARN] [1737143592.044540]: Plan exhausted +[WARN] [1737143592.086550]: Plan exhausted +[WARN] [1737143592.126528]: Plan exhausted +[INFO] [1737143595.784620]: reached home +[INFO] [1737143595.797822]: reset action history +[INFO] [1737143598.751502]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143602.261432]: resetting environment +[INFO] [1737143602.278848]: cleared current plan +[INFO] [1737143602.299229]: moving to home +[INFO] [1737143603.303632]: reached home +[INFO] [1737143603.311521]: reset action history +[INFO] [1737143606.192616]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_7_2025-01-17-14-53-26.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:44003/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2518695] +process[print_text-23]: started with pid [2518696] +process[rosbag_record-24]: started with pid [2518697] +started bagging! +[INFO] [1737143633.355353]: Storing episode... +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit[print_text-23] killing on exit +[INFO] [1737143634.196529]: Stored episode 8. +[INFO] [1737143634.204660]: Saving video... +[INFO] [1737143634.529393]: Video saved! +Starting episode 8 +[INFO] [1737143634.599683]: resetting environment +[INFO] [1737143634.643845]: cleared current plan +[INFO] [1737143634.649416]: moving to home +[INFO] [1737143638.272565]: reached home +[INFO] [1737143638.278846]: reset action history +[INFO] [1737143641.228173]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143644.395193]: resetting environment +[INFO] [1737143644.433173]: cleared current plan +[INFO] [1737143644.452673]: moving to home +[INFO] [1737143645.465311]: reached home +[INFO] [1737143645.470369]: reset action history +[INFO] [1737143648.377440]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_8_2025-01-17-14-54-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:34911/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2518829] +process[print_text-26]: started with pid [2518830] +process[rosbag_record-27]: started with pid [2518831] +started bagging! +For topic gripper_width: timestamp difference is 102142439 for nearest: 1737143655443934546 - target: 1737143655341792107 at idx 111 +For topic gripper_width: timestamp difference is 105369312 for nearest: 1737143657043928844 - target: 1737143656938559532 at idx 133 +For topic gripper_width: timestamp difference is 105739588 for nearest: 1737143657243897909 - target: 1737143657138158321 at idx 135 +For topic gripper_width: timestamp difference is 110657995 for nearest: 1737143659643945997 - target: 1737143659533288002 at idx 168 +[INFO] [1737143679.610431]: Storing episode... +[pub_text-25] killing on exit[print_text-26] killing on exit +[rosbag_record-27] killing on exit +[INFO] [1737143680.413269]: Stored episode 9. +[INFO] [1737143680.427992]: Saving video... +[WARN] [1737143680.765974]: Plan exhausted +[WARN] [1737143680.805308]: Plan exhausted +[WARN] [1737143680.844050]: Plan exhausted +[WARN] [1737143680.885216]: Plan exhausted +[WARN] [1737143680.924622]: Plan exhausted +[WARN] [1737143680.965495]: Plan exhausted +[WARN] [1737143681.004092]: Plan exhausted +[WARN] [1737143681.044615]: Plan exhausted +[WARN] [1737143681.093165]: Plan exhausted +[WARN] [1737143681.126848]: Plan exhausted +[WARN] [1737143681.165029]: Plan exhausted +[WARN] [1737143681.205170]: Plan exhausted +[WARN] [1737143681.246302]: Plan exhausted +[INFO] [1737143681.075696]: Video saved! +Starting episode 9 +[INFO] [1737143681.215672]: resetting environment +[INFO] [1737143681.274101]: cleared current plan +[INFO] [1737143681.281029]: moving to home +[INFO] [1737143684.892072]: reached home +[INFO] [1737143684.901634]: reset action history +[INFO] [1737143687.962653]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143689.540165]: resetting environment +[INFO] [1737143689.606602]: cleared current plan +[INFO] [1737143689.620417]: moving to home +[INFO] [1737143690.634299]: reached home +[INFO] [1737143690.645813]: reset action history +[INFO] [1737143693.641789]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_9_2025-01-17-14-54-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:37883/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2518975] +process[print_text-29]: started with pid [2518976] +process[rosbag_record-30]: started with pid [2518977] +started bagging! +For topic gripper_width: timestamp difference is 124962641 for nearest: 1737143719943972183 - target: 1737143719819009542 at idx 249 +[INFO] [1737143725.520055]: Storing episode... +[rosbag_record-30] killing on exit +[pub_text-28] killing on exit +[print_text-29] killing on exit +[INFO] [1737143726.372968]: Stored episode 10. +[INFO] [1737143726.386262]: Saving video... +[WARN] [1737143726.886460]: Plan exhausted +[WARN] [1737143726.933941]: Plan exhausted +[WARN] [1737143726.964793]: Plan exhausted +[INFO] [1737143726.787376]: Video saved! +Starting episode 10 +[INFO] [1737143726.928096]: resetting environment +[INFO] [1737143726.973043]: cleared current plan +[INFO] [1737143726.978286]: moving to home +[INFO] [1737143730.595901]: reached home +[INFO] [1737143730.602735]: reset action history +[INFO] [1737143733.659205]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143737.885932]: resetting environment +[INFO] [1737143737.941744]: cleared current plan +[INFO] [1737143737.957102]: moving to home +[INFO] [1737143738.977648]: reached home +[INFO] [1737143738.986309]: reset action history +[INFO] [1737143742.008898]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_10_2025-01-17-14-55-42.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:38517/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2519116] +process[print_text-32]: started with pid [2519117] +process[rosbag_record-33]: started with pid [2519118] +started bagging! +[INFO] [1737143772.670119]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[WARN] [1737143774.015656]: Plan exhausted +[WARN] [1737143774.051326]: Plan exhausted +[WARN] [1737143774.093610]: Plan exhausted +[INFO] [1737143773.464463]: Stored episode 11. +[INFO] [1737143773.474484]: Saving video... +[INFO] [1737143773.988003]: Video saved! +Starting episode 11 +[INFO] [1737143774.078788]: resetting environment +[INFO] [1737143774.109237]: cleared current plan +[INFO] [1737143774.119287]: moving to home +[INFO] [1737143777.728066]: reached home +[INFO] [1737143777.738788]: reset action history +[INFO] [1737143780.814029]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143785.289284]: resetting environment +[INFO] [1737143785.321392]: cleared current plan +[INFO] [1737143785.321843]: moving to home +[INFO] [1737143786.324088]: reached home +[INFO] [1737143786.347303]: reset action history +[INFO] [1737143789.383253]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_11_2025-01-17-14-56-29.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:41913/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2519259] +process[print_text-35]: started with pid [2519260] +process[rosbag_record-36]: started with pid [2519261] +started bagging! +For topic gripper_width: timestamp difference is -240954499 for nearest: 1737143803177277465 - target: 1737143803418231964 at idx 201 +[INFO] [1737143822.284637]: Storing episode... +[rosbag_record-36] killing on exit +[pub_text-34] killing on exit +[print_text-35] killing on exit +[WARN] [1737143823.644089]: Plan exhausted +[WARN] [1737143823.690209]: Plan exhausted +[WARN] [1737143823.724825]: Plan exhausted +[WARN] [1737143823.764830]: Plan exhausted +[WARN] [1737143823.805135]: Plan exhausted +[INFO] [1737143823.160585]: Stored episode 12. +[INFO] [1737143823.168683]: Saving video... +[INFO] [1737143823.659507]: Video saved! +Starting episode 12 +[INFO] [1737143823.789918]: resetting environment +[INFO] [1737143823.839273]: cleared current plan +[INFO] [1737143823.846644]: moving to home +[INFO] [1737143827.461584]: reached home +[INFO] [1737143827.464828]: reset action history +[INFO] [1737143830.548813]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143833.504999]: resetting environment +[INFO] [1737143833.561354]: cleared current plan +[INFO] [1737143833.574633]: moving to home +[INFO] [1737143834.585646]: reached home +[INFO] [1737143834.612390]: reset action history +[INFO] [1737143837.677973]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_12_2025-01-17-14-57-17.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:44637/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2519403] +process[print_text-38]: started with pid [2519404] +process[rosbag_record-39]: started with pid [2519405] +started bagging! +For topic gripper_width: timestamp difference is -165962330 for nearest: 1737143854877257236 - target: 1737143855043219566 at idx 235 +For topic gripper_width: timestamp difference is -162430712 for nearest: 1737143866877275280 - target: 1737143867039705992 at idx 249 +[INFO] [1737143870.312178]: Storing episode... +[rosbag_record-39] killing on exit[print_text-38] killing on exit +[pub_text-37] killing on exit +[INFO] [1737143871.135775]: Stored episode 13. +[INFO] [1737143871.157142]: Saving video... +[INFO] [1737143871.371162]: Video saved! +Starting episode 13 +[INFO] [1737143871.512495]: resetting environment +[INFO] [1737143871.535070]: cleared current plan +[INFO] [1737143871.535351]: moving to home +[INFO] [1737143875.146301]: reached home +[INFO] [1737143875.165253]: reset action history +[INFO] [1737143878.278586]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143882.012283]: resetting environment +[INFO] [1737143882.069207]: cleared current plan +[INFO] [1737143882.092222]: moving to home +[INFO] [1737143883.114925]: reached home +[INFO] [1737143883.129030]: reset action history +[INFO] [1737143886.161766]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_13_2025-01-17-14-58-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:41365/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2519534] +process[print_text-41]: started with pid [2519535] +process[rosbag_record-42]: started with pid [2519536] +started bagging! +[INFO] [1737143920.795845]: Storing episode... +[rosbag_record-42] killing on exit +[pub_text-40] killing on exit +[print_text-41] killing on exit +[WARN] [1737143922.124529]: Plan exhausted +[WARN] [1737143922.168931]: Plan exhausted +[WARN] [1737143922.204271]: Plan exhausted +[WARN] [1737143922.245391]: Plan exhausted +[WARN] [1737143922.284452]: Plan exhausted +[INFO] [1737143921.704904]: Stored episode 14. +[INFO] [1737143921.711260]: Saving video... +[INFO] [1737143922.146172]: Video saved! +Starting episode 14 +[INFO] [1737143922.291446]: resetting environment +[INFO] [1737143922.352192]: cleared current plan +[INFO] [1737143922.358633]: moving to home +[WARN] [1737143922.332361]: Plan exhausted +[INFO] [1737143925.974780]: reached home +[INFO] [1737143926.008006]: reset action history +[INFO] [1737143929.238503]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143930.400680]: resetting environment +[INFO] [1737143930.473414]: cleared current plan +[INFO] [1737143930.492514]: moving to home +[INFO] [1737143931.512691]: reached home +[INFO] [1737143931.535982]: reset action history +[INFO] [1737143934.624734]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_14_2025-01-17-14-58-54.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:39485/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2519679] +process[print_text-44]: started with pid [2519680] +process[rosbag_record-45]: started with pid [2519681] +started bagging! +[INFO] [1737143969.977056]: Storing episode... +[rosbag_record-45] killing on exit +[print_text-44] killing on exit +[pub_text-43] killing on exit +[INFO] [1737143970.891929]: Stored episode 15. +[INFO] [1737143970.918944]: Saving video... +[WARN] [1737143971.333604]: Plan exhausted +[WARN] [1737143971.369226]: Plan exhausted +[WARN] [1737143971.413410]: Plan exhausted +[WARN] [1737143971.444073]: Plan exhausted +[WARN] [1737143971.486899]: Plan exhausted +[WARN] [1737143971.539267]: Plan exhausted +[WARN] [1737143971.567342]: Plan exhausted +[WARN] [1737143971.606930]: Plan exhausted +[WARN] [1737143971.644071]: Plan exhausted +[WARN] [1737143971.685118]: Plan exhausted +[WARN] [1737143971.731335]: Plan exhausted +[WARN] [1737143971.768925]: Plan exhausted +[WARN] [1737143971.804368]: Plan exhausted +[INFO] [1737143971.590230]: Video saved! +Starting episode 15 +[INFO] [1737143971.751004]: resetting environment +[INFO] [1737143971.808055]: cleared current plan +[INFO] [1737143971.811193]: moving to home +[INFO] [1737143975.421074]: reached home +[INFO] [1737143975.442260]: reset action history +[INFO] [1737143978.626080]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737143982.324393]: resetting environment +[INFO] [1737143982.390602]: cleared current plan +[INFO] [1737143982.396579]: moving to home +[INFO] [1737143983.428306]: reached home +[INFO] [1737143983.432729]: reset action history +[INFO] [1737143986.645727]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_15_2025-01-17-14-59-46.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:41977/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2519811] +process[print_text-47]: started with pid [2519812] +process[rosbag_record-48]: started with pid [2519813] +started bagging! +For topic gripper_width: timestamp difference is -101387638 for nearest: 1737143987577314478 - target: 1737143987678702116 at idx 26 +[INFO] [1737144013.938001]: Storing episode... +[rosbag_record-48] killing on exit +[pub_text-46] killing on exit +[print_text-47] killing on exit +[INFO] [1737144014.771702]: Stored episode 16. +[INFO] [1737144014.779838]: Saving video... +[WARN] [1737144015.290921]: Plan exhausted +[WARN] [1737144015.325742]: Plan exhausted +[WARN] [1737144015.365034]: Plan exhausted +[WARN] [1737144015.409395]: Plan exhausted +[WARN] [1737144015.447976]: Plan exhausted +[WARN] [1737144015.484870]: Plan exhausted +[WARN] [1737144015.536699]: Plan exhausted +[WARN] [1737144015.566553]: Plan exhausted +[WARN] [1737144015.606638]: Plan exhausted +[WARN] [1737144015.644937]: Plan exhausted +[WARN] [1737144015.694937]: Plan exhausted +[INFO] [1737144015.424219]: Video saved! +Starting episode 16 +[INFO] [1737144015.670635]: resetting environment +[INFO] [1737144015.716384]: cleared current plan +[INFO] [1737144015.733932]: moving to home +[INFO] [1737144019.354322]: reached home +[INFO] [1737144019.380682]: reset action history +[INFO] [1737144022.556634]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144023.787925]: resetting environment +[INFO] [1737144023.900355]: cleared current plan +[INFO] [1737144023.912735]: moving to home +[INFO] [1737144024.928058]: reached home +[INFO] [1737144024.975122]: reset action history +[INFO] [1737144028.199858]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_16_2025-01-17-15-00-28.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:46219/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2519955] +process[print_text-50]: started with pid [2519956] +process[rosbag_record-51]: started with pid [2519957] +started bagging! +[INFO] [1737144064.695148]: Storing episode... +[pub_text-49] killing on exit +[rosbag_record-51] killing on exit +[print_text-50] killing on exit +[WARN] [1737144066.044098]: Plan exhausted +[WARN] [1737144066.092210]: Plan exhausted +[WARN] [1737144066.131527]: Plan exhausted +[WARN] [1737144066.165402]: Plan exhausted +[WARN] [1737144066.204058]: Plan exhausted +[WARN] [1737144066.263774]: Plan exhausted +[WARN] [1737144066.292299]: Plan exhausted +[INFO] [1737144065.571110]: Stored episode 17. +[INFO] [1737144065.581607]: Saving video... +[INFO] [1737144066.059907]: Video saved! +Starting episode 17 +[INFO] [1737144066.239103]: resetting environment +[INFO] [1737144066.302737]: cleared current plan +[INFO] [1737144066.302930]: moving to home +[INFO] [1737144069.934088]: reached home +[INFO] [1737144069.947159]: reset action history +[INFO] [1737144073.270121]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144074.124890]: resetting environment +[INFO] [1737144074.176657]: cleared current plan +[INFO] [1737144074.189215]: moving to home +[INFO] [1737144075.197152]: reached home +[INFO] [1737144075.217149]: reset action history +[INFO] [1737144078.437331]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_17_2025-01-17-15-01-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:39817/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2520065] +process[print_text-53]: started with pid [2520089] +process[rosbag_record-54]: started with pid [2520090] +started bagging! +For topic gripper_width: timestamp difference is 185384531 for nearest: 1737144083677278538 - target: 1737144083491894007 at idx 75 +For topic gripper_width: timestamp difference is -130666935 for nearest: 1737144104877239740 - target: 1737144105007906675 at idx 249 +[INFO] [1737144105.471650]: Storing episode... +[pub_text-52] killing on exit +[print_text-53] killing on exit[rosbag_record-54] killing on exit +[INFO] [1737144106.973022]: Stored episode 18. +[INFO] [1737144107.016712]: Saving video... +[WARN] [1737144106.809612]: Plan exhausted +[WARN] [1737144106.853784]: Plan exhausted +[WARN] [1737144106.891950]: Plan exhausted +[WARN] [1737144106.958132]: Plan exhausted +[WARN] [1737144107.011476]: Plan exhausted +[WARN] [1737144107.058261]: Plan exhausted +[WARN] [1737144107.087877]: Plan exhausted +[WARN] [1737144107.107287]: Plan exhausted +[WARN] [1737144107.129830]: Plan exhausted +[WARN] [1737144107.173405]: Plan exhausted +[WARN] [1737144107.217362]: Plan exhausted +[WARN] [1737144107.244977]: Plan exhausted +[WARN] [1737144107.284542]: Plan exhausted +[WARN] [1737144107.343736]: Plan exhausted +[WARN] [1737144107.366045]: Plan exhausted +[WARN] [1737144107.408817]: Plan exhausted +[WARN] [1737144107.461956]: Plan exhausted +[WARN] [1737144107.491346]: Plan exhausted +[WARN] [1737144107.524562]: Plan exhausted +[WARN] [1737144107.564092]: Plan exhausted +[WARN] [1737144107.610906]: Plan exhausted +[WARN] [1737144107.647196]: Plan exhausted +[WARN] [1737144107.685730]: Plan exhausted +[WARN] [1737144107.725427]: Plan exhausted +[WARN] [1737144107.768481]: Plan exhausted +[WARN] [1737144107.807545]: Plan exhausted +[WARN] [1737144107.854129]: Plan exhausted +[WARN] [1737144107.886202]: Plan exhausted +[INFO] [1737144107.634666]: Video saved! +Starting episode 18 +[INFO] [1737144107.844279]: resetting environment +[INFO] [1737144107.898394]: cleared current plan +[INFO] [1737144107.901758]: moving to home +[INFO] [1737144111.509214]: reached home +[INFO] [1737144111.537524]: reset action history +[INFO] [1737144114.918549]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144120.784911]: resetting environment +[INFO] [1737144120.903079]: cleared current plan +[INFO] [1737144120.912699]: moving to home +[INFO] [1737144121.939949]: reached home +[INFO] [1737144121.941144]: reset action history +[INFO] [1737144125.151052]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_18_2025-01-17-15-02-05.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:36701/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2520233] +process[print_text-56]: started with pid [2520234] +process[rosbag_record-57]: started with pid [2520235] +started bagging! +[INFO] [1737144157.986474]: Storing episode... +[rosbag_record-57] killing on exit +[print_text-56] killing on exit +[pub_text-55] killing on exit +[INFO] [1737144158.855236]: Stored episode 19. +[INFO] [1737144158.871405]: Saving video... +[WARN] [1737144159.343145]: Plan exhausted +[WARN] [1737144159.368431]: Plan exhausted +[WARN] [1737144159.406341]: Plan exhausted +[WARN] [1737144159.444548]: Plan exhausted +[WARN] [1737144159.505192]: Plan exhausted +[WARN] [1737144159.533491]: Plan exhausted +[INFO] [1737144159.276861]: Video saved! +Starting episode 19 +[INFO] [1737144159.482871]: resetting environment +[INFO] [1737144159.559007]: cleared current plan +[INFO] [1737144159.563091]: moving to home +[INFO] [1737144163.214904]: reached home +[INFO] [1737144163.239574]: reset action history +[INFO] [1737144166.514701]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737144167.941295]: resetting environment +[INFO] [1737144167.992389]: cleared current plan +[INFO] [1737144167.992778]: moving to home +[INFO] [1737144168.996522]: reached home +[INFO] [1737144169.008272]: reset action history +[INFO] [1737144172.271948]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/episode_rosbags/episode_19_2025-01-17-15-02-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2517388.log +started roslaunch server http://158.130.50.37:36007/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2520380] +process[print_text-59]: started with pid [2520381] +process[rosbag_record-60]: started with pid [2520382] +started bagging! +For topic gripper_width: timestamp difference is 100854938 for nearest: 1737144185143942420 - target: 1737144185043087482 at idx 172 +For topic gripper_width: timestamp difference is 101468771 for nearest: 1737144185943940609 - target: 1737144185842471838 at idx 182 +For topic gripper_width: timestamp difference is 102735824 for nearest: 1737144187543967313 - target: 1737144187441231489 at idx 206 +For topic gripper_width: timestamp difference is 103318143 for nearest: 1737144188343943286 - target: 1737144188240625143 at idx 218 +For topic gripper_width: timestamp difference is 104589251 for nearest: 1737144189943965939 - target: 1737144189839376688 at idx 243 +For topic gripper_width: timestamp difference is 104744592 for nearest: 1737144190143966832 - target: 1737144190039222240 at idx 245 +For topic gripper_width: timestamp difference is 105179536 for nearest: 1737144190743939626 - target: 1737144190638760090 at idx 249 +For topic gripper_width: timestamp difference is 106428721 for nearest: 1737144192343898394 - target: 1737144192237469673 at idx 249 +For topic gripper_width: timestamp difference is 107083708 for nearest: 1737144193143903881 - target: 1737144193036820173 at idx 249 +For topic gripper_width: timestamp difference is 107715464 for nearest: 1737144193943890667 - target: 1737144193836175203 at idx 249 +For topic gripper_width: timestamp difference is 115712563 for nearest: 1737144204943941514 - target: 1737144204828228951 at idx 249 +For topic gripper_width: timestamp difference is 116142122 for nearest: 1737144205143973200 - target: 1737144205027831078 at idx 249 +For topic gripper_width: timestamp difference is 117337322 for nearest: 1737144205743900633 - target: 1737144205626563311 at idx 249 +For topic gripper_width: timestamp difference is 119404601 for nearest: 1737144206743939685 - target: 1737144206624535084 at idx 249 +[INFO] [1737144209.909790]: Storing episode... +[rosbag_record-60] killing on exit +[print_text-59] killing on exit +[pub_text-58] killing on exit +[INFO] [1737144210.812712]: Stored episode 20. +[INFO] [1737144210.839331]: Saving video... +[WARN] [1737144211.094123]: Plan exhausted +[WARN] [1737144211.125738]: Plan exhausted +[WARN] [1737144211.166485]: Plan exhausted +[WARN] [1737144211.214847]: Plan exhausted +[WARN] [1737144211.272771]: Plan exhausted +[WARN] [1737144211.288854]: Plan exhausted +[WARN] [1737144211.325918]: Plan exhausted +[WARN] [1737144211.376716]: Plan exhausted +[WARN] [1737144211.405338]: Plan exhausted +[WARN] [1737144211.444062]: Plan exhausted +[WARN] [1737144211.489571]: Plan exhausted +[WARN] [1737144211.525994]: Plan exhausted +[WARN] [1737144211.564086]: Plan exhausted +[WARN] [1737144211.605885]: Plan exhausted +[WARN] [1737144211.671332]: Plan exhausted +[WARN] [1737144211.695162]: Plan exhausted +[INFO] [1737144211.441018]: Video saved! +[INFO] [1737144211.642866]: resetting environment +[INFO] [1737144211.726072]: cleared current plan +[INFO] [1737144211.731548]: moving to home +[INFO] [1737144215.384654]: reached home +[INFO] [1737144215.406573]: reset action history +[INFO] [1737144218.869083]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt b/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..ed960f9ae1689f11a78f267e474d398511995733 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,95 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..a6b1973 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d9b15a5 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json b/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7b1a15353b61772e6d68d9822f622d7d68c6d00a --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-17T19:47:42.282982", + "startedAt": "2025-01-17T19:47:41.909118", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4134.258, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.962, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 399.98992919921875 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json b/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..234c8261410960139df0739cff1fd3ca6afda240 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "1819f930b352d4eea54cb805305cebbd6f2ce37f51bd15e133ca4ef3bf2aea7a", "size": 1144404, "path": "media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4"}, "global_step": 2658, "_timestamp": 1737144221.4700227, "_runtime": 959.5483906269073, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "093e197b5aca9677627b2fe474f40045296961c6d67cd343459930ef0fdfea31", "size": 1188450, "path": "media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4"}, "eval/num_success": 20.0, "episode": 80.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8014022707939148, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.837080717086792, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.934773862361908, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.9148585796356201, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "ee6ae32e8cb4b17611f28e875305214f9d7b534d2c4eb2e243db0f4b59e6a53f", "size": 1146552, "path": "media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ccfd7c2808a5399343dba4263c583d12e3f79c291c8523c1ac43c6dbbce9a53b", "size": 1148831, "path": "media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "916d91435e19178c30f6ad3ccb21d41fcb4f5584d80aea0864a22682c8d60b5f", "size": 1159225, "path": "media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "abbb012eb02e7137181fe4902ad630dfc552a4d8a10b02437e92852077a2c37e", "size": 1229823, "path": "media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "a92f6e4d8653d4439383f6af37f6414a03e15252e2ef97c40394e947687cc0e5", "size": 1162123, "path": "media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "baeae50ab773d27ae44b9d6fd2c8a77836db35a896dec3de71b24adc6c06a9bc", "size": 1153241, "path": "media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "f6383c0f370fe43d88dd9487aa2c5e8fccf8ac1b03a10253552f375d060493ea", "size": 1317618, "path": "media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "a200f16787e6024f65815a1c43b940d619d7f92303cb6b423b1d4573f8a4c2f8", "size": 1343379, "path": "media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "268ff9e2e53b41188aa0fa482c7bfb19f2fb2785f83d12074c59151944a43b98", "size": 1293306, "path": "media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "30dc16b787631229f3663c1d3202233d965a1bc38599211ed848ed4abdd882da", "size": 1370113, "path": "media/videos/eval/11_eval_11_30dc16b787631229f366.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "79b7fe799da0fe5e29e5c659a71124a879c60c4ccf2e14eee9ca19e62f92aa6d", "size": 1367490, "path": "media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "72586beaede9059dffaad451ccb47b783d6a63ddc277e0544b3f874c0377af41", "size": 1447624, "path": "media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "bfd6e56a3506bb346792fc8d7c11910474f4099601c968e0484ff1921e7565a1", "size": 1458401, "path": "media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "9e02939fb08b9aeb13e66076664fc0010c1e76536ca48fa048578a2b52b106ed", "size": 1167199, "path": "media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "d19b3abe759fefca682750d3714304bc8492a89c5fc8d582c7add5e46b26341d", "size": 1515039, "path": "media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "9039e216f093cde1d3f441956a295f944bc8db517e4fbe27227dde92cf249a72", "size": 1188164, "path": "media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "d73f4283df1c3c229b048e1ca89694edabce51c187862a795a376e5126820abb", "size": 1381842, "path": "media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "1265d28d813382198c39f4d4d8adce2a6ae621c1dce45866f8e1475c1d56acb7", "size": 1572862, "path": "media/videos/eval/19_eval_19_1265d28d813382198c39.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "fc42d684313cd17be1f80cb486db2bb46903619eacd3baffbd28a986ac01a4f4", "size": 330, "artifact_path": "wandb-client-artifact://oc2anruc3699luifmfrjlvujuaw67j0tc1h4mum637tq8jeayh2ijwp2ccrvz3c8pazjwhnumzlwa0tbday9s11vp7189jwe1nc4llwfdy1u37cmyqau0rhdby9w9ord/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2u3ytze84ucjj8qur2bt6xh3lnessibbe4hfioskk7w7gh2min2wcro5dt0j2mresv40w6stfo3lksca99bqc2k4vxyrwbjc7dgbfvckt0n75cncw6y78d2gyym4m73l:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 66, "total_num_episodes": 80, "total_success_rate": 0.825, "total_success_rate_lb_0.01": 0.7136867939998497, "total_success_rate_ub_0.01": 0.9083331971543944, "total_success_rate_lb_0.025": 0.7333697080145443, "total_success_rate_ub_0.025": 0.8928065522719597, "total_success_rate_lb_0.05": 0.7536826841025817, "total_success_rate_ub_0.05": 0.8825706876049224, "total_success_rate_lb_0.1": 0.7583227343340642, "total_success_rate_ub_0.1": 0.875470220967539, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "37909e05a422137136b701edb4455ffcfe6905d91d0702c7bd93db4963f9cb2f", "size": 506, "artifact_path": "wandb-client-artifact://34hyq7uflmilxt3zo7j2mg2twcsqijo4to3pdo1vvgcw0bg14cyib7hldzssikeb7l95898ykc7di6h2aky3a6c44x99asghiq5cbh073aczdutrzrfyx7ca5wd6cxn2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://i01ge07kcfnxz8qq89b0qo4vyir1rsdeylu3he51qnq11c5xs7q5p6gxewbc0aolwkq5voa68yyeatvmqtsfpspnawykfbsbje4w1dmmbxgsrh840zswsnuhlmtow9y0:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 980}, "eval/total_success_rate": 0.824999988079071, "eval/total_success_rate_lb_0.01": 0.7136867642402649, "eval/total_success_rate_ub_0.01": 0.9083331823348999, "eval/total_success_rate_lb_0.025": 0.7333697080612183, "eval/total_success_rate_ub_0.025": 0.8928065299987793, "eval/total_success_rate_lb_0.05": 0.7536826729774475, "eval/total_success_rate_ub_0.05": 0.8825706839561462, "eval/total_success_rate_lb_0.1": 0.7583227157592773, "eval/total_success_rate_ub_0.1": 0.8754702210426331} \ No newline at end of file diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug-internal.log b/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7007748425f1f183ab7912bfb66b8364bf1a3d39 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug-internal.log @@ -0,0 +1,2141 @@ +2025-01-17 14:47:41,921 INFO StreamThr :2517552 [internal.py:wandb_internal():85] W&B internal server running at pid: 2517552, started at: 2025-01-17 14:47:41.921158 +2025-01-17 14:47:41,922 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status +2025-01-17 14:47:41,933 INFO WriterThread:2517552 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb +2025-01-17 14:47:41,934 DEBUG SenderThread:2517552 [sender.py:send():379] send: header +2025-01-17 14:47:41,942 DEBUG SenderThread:2517552 [sender.py:send():379] send: run +2025-01-17 14:47:42,165 INFO SenderThread:2517552 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files +2025-01-17 14:47:42,165 INFO SenderThread:2517552 [sender.py:_start_run_threads():1188] run started: up07mbix with start time 1737143261.921632 +2025-01-17 14:47:42,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 14:47:42,182 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: check_version +2025-01-17 14:47:42,265 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 14:47:42,270 DEBUG HandlerThread:2517552 [system_info.py:__init__():26] System info init +2025-01-17 14:47:42,270 DEBUG HandlerThread:2517552 [system_info.py:__init__():41] System info init done +2025-01-17 14:47:42,270 INFO HandlerThread:2517552 [system_monitor.py:start():194] Starting system monitor +2025-01-17 14:47:42,270 INFO SystemMonitor:2517552 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 14:47:42,270 INFO HandlerThread:2517552 [system_monitor.py:probe():214] Collecting system info +2025-01-17 14:47:42,271 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 14:47:42,271 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started disk monitoring +2025-01-17 14:47:42,272 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 14:47:42,272 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started memory monitoring +2025-01-17 14:47:42,273 INFO SystemMonitor:2517552 [interfaces.py:start():188] Started network monitoring +2025-01-17 14:47:42,282 DEBUG HandlerThread:2517552 [system_info.py:probe():152] Probing system +2025-01-17 14:47:42,286 DEBUG HandlerThread:2517552 [system_info.py:_probe_git():137] Probing git +2025-01-17 14:47:42,292 DEBUG HandlerThread:2517552 [system_info.py:_probe_git():145] Probing git done +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_info.py:probe():200] Probing system done +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T19:47:42.282982', 'startedAt': '2025-01-17T19:47:41.909118', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4134.258, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 399.98992919921875}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 14:47:42,293 INFO HandlerThread:2517552 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 14:47:42,293 INFO HandlerThread:2517552 [system_monitor.py:probe():227] Publishing system info +2025-01-17 14:47:42,293 DEBUG HandlerThread:2517552 [system_info.py:_save_code():44] Saving code +2025-01-17 14:47:42,294 DEBUG HandlerThread:2517552 [system_info.py:_save_code():67] Saving code done +2025-01-17 14:47:42,294 DEBUG HandlerThread:2517552 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 14:47:42,390 DEBUG HandlerThread:2517552 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 14:47:42,390 INFO HandlerThread:2517552 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 14:47:42,393 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:47:42,393 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 14:47:42,394 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 14:47:42,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 14:47:42,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: python_packages +2025-01-17 14:47:42,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:47:42,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:47:42,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:42,655 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/5fdjf9gk-code/FISH/eval_robot.py +2025-01-17 14:47:42,664 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/9ted8sye-wandb-metadata.json +2025-01-17 14:47:42,669 INFO HandlerThread:2517552 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/tb" + save: true +} +control { + relay_id: "ac8462253926464da2d1680d5c59cfdf" +} +_info { + stream_id: "up07mbix" +} + +2025-01-17 14:47:42,669 DEBUG HandlerThread:2517552 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 14:47:42,688 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/aoalkeoe-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:47:42,689 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/kci0a5wf-diff.patch +2025-01-17 14:47:42,691 DEBUG SenderThread:2517552 [sender.py:send():379] send: telemetry +2025-01-17 14:47:42,737 DEBUG SenderThread:2517552 [sender.py:send():379] send: tbrecord +2025-01-17 14:47:42,738 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:47:42,738 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 with policy live +2025-01-17 14:47:42,919 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/85ejh0y0-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:47:43,166 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH +2025-01-17 14:47:43,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code +2025-01-17 14:47:43,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:44,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:45,166 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:45,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:46,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:47,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:47,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:48,084 DEBUG SenderThread:2517552 [sender.py:send():379] send: config +2025-01-17 14:47:48,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:49,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:49,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:50,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:51,167 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:51,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:52,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:52,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:53,168 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:53,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:54,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:55,169 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:55,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:56,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:57,169 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:47:57,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:47:57,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:47:57,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:57,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:47:58,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:47:59,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:00,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:01,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:02,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:03,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:03,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:04,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:05,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:06,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:07,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:08,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:08,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:09,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:10,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:11,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:12,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:12,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:12,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:13,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:13,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:13,736 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:14,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 14:48:14,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:15,174 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:15,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:16,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:17,175 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:17,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:18,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:19,628 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:19,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:20,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:21,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:22,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:23,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:24,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:25,473 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:25,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:26,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:27,178 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:27,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:27,626 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:27,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:28,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:29,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:30,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:30,776 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:31,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:32,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:33,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:34,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:35,180 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:35,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:36,317 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:36,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:37,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:38,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:39,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:40,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:41,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:41,983 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:42,172 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:48:42,172 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 with policy now +2025-01-17 14:48:42,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:48:42,174 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:48:42,174 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:48:42,174 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:48:42,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media +2025-01-17 14:48:42,273 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 14:48:42,274 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:48:42,469 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ybp3svcy-media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:42,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:42,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:42,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:42,745 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/lbxqg749-media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 14:48:43,182 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:43,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:44,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:45,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:46,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:47,184 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:47,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:47,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:48,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:49,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:50,636 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:51,185 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:51,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:52,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:53,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:53,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:54,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:55,186 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:48:55,444 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/a46a0ts1-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:48:55,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:56,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:57,187 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:57,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:48:57,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:48:57,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:58,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:48:59,188 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:48:59,383 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:48:59,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:00,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:01,188 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:01,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:02,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:03,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:04,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:04,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:05,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:06,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:07,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:08,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:09,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:09,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:10,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:11,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:12,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:49:12,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:12,627 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:12,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:13,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:14,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:14,751 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:15,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:16,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:17,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:18,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:19,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:20,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:20,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:21,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:22,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:23,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:24,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:25,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:25,862 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:26,270 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:49:26,270 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 with policy now +2025-01-17 14:49:26,273 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:49:26,273 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:49:26,274 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:49:26,274 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:49:26,525 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hg6wq3ey-media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:26,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:27,198 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:49:27,484 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/b3bqb9i9-media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 14:49:27,634 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:27,634 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:27,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:28,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:29,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:30,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:30,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:31,199 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:31,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:32,665 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:33,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:34,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:35,201 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:35,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:36,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:36,666 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:37,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:38,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:39,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:40,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:41,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:42,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:42,275 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:49:42,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:42,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:42,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:43,142 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:49:43,142 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:49:43,142 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:49:43,203 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:49:43,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:43,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:49:43,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:44,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:45,204 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:45,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:46,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:47,205 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:49:47,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:47,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:48,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:49,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:50,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:51,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:52,632 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:52,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:53,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:54,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:55,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:56,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:57,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:49:57,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:49:57,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:57,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:49:58,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:49:59,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:00,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:01,210 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:01,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:02,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:03,211 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:03,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:03,656 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:04,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:05,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:06,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:07,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:08,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:09,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:09,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:10,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:11,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:12,275 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:50:12,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:12,629 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:12,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:13,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:14,407 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:50:14,408 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 with policy now +2025-01-17 14:50:14,410 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:50:14,410 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:14,411 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:14,411 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:14,658 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:14,665 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/cnvywjy0-media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:15,216 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:50:15,428 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:15,501 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/kgs37oz6-media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 14:50:15,660 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:16,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:17,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:17,669 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:18,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:19,217 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:19,671 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:20,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:20,823 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:21,676 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:22,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:23,219 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:23,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:24,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:25,676 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:26,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:26,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:27,220 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:50:27,490 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/mkcw0g0c-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:50:27,633 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:27,633 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:27,677 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:27,938 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:27,938 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:27,938 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:28,221 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:28,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:29,221 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:29,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:30,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:31,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:32,139 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:32,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:33,222 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:33,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:34,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:35,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:36,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:37,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:37,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:38,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:39,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:40,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:41,225 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:41,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:42,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:50:42,629 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:42,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:42,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:42,726 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:43,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:44,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:45,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:46,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:47,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:48,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:48,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:49,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:50,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:51,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:52,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:53,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:54,638 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:50:54,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:55,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:56,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:57,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:50:57,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:50:57,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:58,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:50:59,137 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:50:59,137 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:50:59,138 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 with policy now +2025-01-17 14:50:59,138 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:50:59,138 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:50:59,138 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:50:59,232 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,233 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:50:59,390 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ma9sq65m-media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,665 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/8lbemu2o-media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 14:50:59,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:00,165 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:00,693 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:01,233 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:01,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:02,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:03,685 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:04,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:05,234 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:05,565 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:05,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:06,688 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:07,235 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:07,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:08,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:09,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:10,642 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:10,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:11,695 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:11,992 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:11,993 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:11,993 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:12,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:12,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:12,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:51:12,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:12,641 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:12,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:13,237 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:13,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:14,699 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:15,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:16,546 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:16,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:17,239 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:17,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:18,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:19,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:20,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:21,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:21,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:22,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:23,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:24,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:25,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:26,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:26,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:27,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:27,630 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:27,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:28,696 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:29,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:30,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:31,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:31,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:32,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:33,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:34,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:35,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:36,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:37,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:37,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:38,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:39,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:40,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:41,698 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:42,276 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:51:42,631 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:42,631 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:42,706 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:42,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:43,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:43,381 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:51:43,382 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 with policy now +2025-01-17 14:51:43,386 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:51:43,387 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:43,387 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:43,388 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:43,632 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ixvd3p6b-media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:43,708 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:44,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:51:44,551 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qmgc5aka-media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 14:51:44,708 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:45,248 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:45,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:46,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:47,712 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:48,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:48,717 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:49,250 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:49,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:50,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:51,251 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:51,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:52,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:53,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:54,164 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:51:54,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:54,512 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/r2k4us42-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:51:54,725 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:55,044 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:51:55,045 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:51:55,045 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:51:55,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:51:55,252 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:55,718 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:56,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:57,253 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:57,637 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:51:57,638 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:51:57,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:58,733 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:59,253 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:51:59,727 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:51:59,964 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:00,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:01,254 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:01,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:02,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:03,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:04,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:05,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:05,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:06,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:07,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:08,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:09,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:10,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:10,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:11,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:12,277 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:52:12,630 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:12,631 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:12,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:13,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:14,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:15,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:15,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:16,721 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:17,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:18,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:19,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:20,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:21,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:21,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:22,722 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:23,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:24,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:25,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:26,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:26,723 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:27,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:27,643 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:27,738 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:28,247 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:52:28,247 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 with policy now +2025-01-17 14:52:28,254 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:52:28,255 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:52:28,255 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:52:28,255 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:52:28,264 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:52:28,264 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:28,265 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:52:28,504 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qdc9zoja-media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:28,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:28,766 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/o5g4fsr7-media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 14:52:29,265 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:29,738 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:30,737 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:31,736 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:31,925 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:32,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:33,266 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:33,744 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:34,751 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:35,750 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:36,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:37,268 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:37,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:37,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:38,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:39,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:40,107 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:52:40,107 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:52:40,108 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:52:40,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:52:40,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:52:40,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:41,269 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:41,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:42,277 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:52:42,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:42,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:42,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:42,777 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:43,270 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:43,758 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:44,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:45,270 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:45,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:46,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:47,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:48,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:48,759 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:49,272 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:49,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:50,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:51,273 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:51,760 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:52,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:53,274 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:52:53,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:53,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:54,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:55,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:56,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:57,635 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:52:57,635 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:52:57,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:58,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:52:58,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:52:59,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:00,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:01,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:02,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:03,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:04,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:04,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:05,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:06,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:07,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:08,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:09,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:10,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:10,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:11,769 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:12,129 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:53:12,129 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 with policy now +2025-01-17 14:53:12,137 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:53:12,138 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:12,138 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:12,141 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:12,278 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:53:12,281 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:12,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:53:12,382 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/md47hjpp-media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:12,645 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:12,665 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/6upv9n67-media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 14:53:12,780 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:13,282 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:13,773 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:14,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:15,283 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:15,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:15,786 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:16,791 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:17,283 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:17,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:18,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:19,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:20,784 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:20,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:21,285 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:21,790 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:22,285 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:53:22,538 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/b3hid05a-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:53:22,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:23,161 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:23,161 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:23,161 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:23,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:23,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:23,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:24,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:25,286 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:25,801 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:26,195 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:26,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:27,287 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:27,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:27,639 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:27,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:28,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:29,288 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:29,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:30,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:31,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:31,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:32,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:33,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:34,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:35,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:36,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:37,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:37,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:38,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:39,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:40,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:41,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:42,278 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:53:42,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:42,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:42,796 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:42,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:43,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:44,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:45,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:46,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:47,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:48,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:48,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:49,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:50,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:51,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:52,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:53,822 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:54,197 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:54,587 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:53:54,587 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 with policy now +2025-01-17 14:53:54,590 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:53:54,590 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:53:54,590 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:53:54,590 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:53:54,810 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:54,847 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ls4gvw2l-media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,298 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:53:55,578 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/pjw07pfa-media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 14:53:55,816 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:56,827 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:53:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:53:57,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:58,839 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:53:59,299 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:53:59,317 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:53:59,842 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:00,845 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:01,838 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:02,847 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:03,301 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:03,836 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:04,357 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:04,844 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:05,213 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:05,213 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:05,213 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:05,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:05,846 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:06,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:07,302 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:07,845 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:08,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:09,303 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:09,429 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:09,857 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:10,865 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:11,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:12,279 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:54:12,649 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:12,649 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:12,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:13,304 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:13,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:14,851 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:14,915 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:15,305 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:15,852 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:16,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:17,306 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:17,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:18,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:19,307 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:19,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:20,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:20,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:21,308 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:21,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:22,853 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:23,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:24,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:25,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:26,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:26,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:27,649 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:27,649 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:27,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:28,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:29,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:30,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:31,726 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:31,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:32,854 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:33,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:34,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:35,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:36,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:37,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:37,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:38,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:39,855 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:40,873 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:41,194 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:54:41,194 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 with policy now +2025-01-17 14:54:41,202 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:54:41,202 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:41,202 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:41,203 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:41,315 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:41,315 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:41,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:54:41,459 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/71638wfj-media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,740 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/yv16vlac-media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 14:54:41,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:42,279 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:54:42,667 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:42,667 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:42,667 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:42,905 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:43,316 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:43,897 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:44,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:45,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:46,896 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:47,317 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:47,680 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:47,901 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:48,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:49,318 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:49,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:50,273 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:54:50,274 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:54:50,274 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:54:50,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:54:50,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:50,589 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/bkkmsaqw-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:54:50,900 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:51,319 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:51,905 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:52,903 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:53,644 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:53,901 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:54,904 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:55,321 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:55,916 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:56,921 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:57,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:54:57,668 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:54:57,922 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:58,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:54:58,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:54:59,322 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:54:59,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:00,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:01,323 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:01,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:02,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:03,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:04,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:04,906 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:05,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:06,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:07,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:08,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:09,668 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:09,907 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:10,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:11,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:12,280 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:55:12,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:12,650 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:12,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:13,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:14,731 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:14,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:15,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:16,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:17,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:18,908 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:19,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:19,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:20,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:21,331 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:21,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:22,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:23,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:24,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:25,520 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:25,934 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:26,898 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:55:26,898 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 with policy now +2025-01-17 14:55:26,912 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:55:26,913 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:55:26,913 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:55:26,913 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:55:26,945 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:27,179 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/zay1i2n7-media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:55:27,614 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/a8q9styt-media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 14:55:27,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:27,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:27,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:28,936 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:29,334 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:29,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:30,599 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:30,949 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:31,335 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:31,971 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:32,934 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:33,932 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:34,950 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:35,337 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:35,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:35,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:36,950 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:37,965 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:38,341 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:55:38,341 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:55:38,342 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:55:38,342 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:55:38,968 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:39,343 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:55:39,343 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:39,952 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:40,953 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:41,025 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:41,344 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:41,976 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:42,280 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:55:42,661 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:42,662 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:42,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:43,344 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:43,954 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:44,962 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:45,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:46,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:47,005 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:47,346 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:55:47,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:48,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:49,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:50,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:51,956 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:52,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:52,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:53,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:54,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:55,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:56,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:55:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:55:57,810 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:55:57,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:58,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:55:59,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:00,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:01,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:02,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:03,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:03,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:04,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:05,958 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:06,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:07,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:08,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:08,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:09,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:10,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:11,959 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:12,281 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:56:12,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:12,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:13,004 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:13,356 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:13,966 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:13,988 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:14,061 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:56:14,061 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 with policy now +2025-01-17 14:56:14,064 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:56:14,064 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:56:14,064 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:56:14,065 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:56:14,336 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ct8nouz2-media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:56:14,632 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ucmgw36v-media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 14:56:14,970 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:15,357 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:15,976 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:16,977 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:17,974 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:18,997 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:19,359 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:19,773 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:19,984 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:20,969 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:21,361 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:21,983 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:22,988 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:23,361 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:24,021 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:25,012 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:25,182 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:25,362 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:56:25,396 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:56:25,397 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:56:25,397 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:56:25,644 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g762glol-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:56:26,012 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:26,363 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:56:27,022 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:27,363 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:27,654 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:27,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:28,011 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:29,016 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:30,013 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:30,428 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:31,048 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:31,365 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:32,043 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:33,051 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:34,051 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:35,056 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:35,483 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:36,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:37,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:37,367 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:38,044 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:39,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:40,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:40,689 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:41,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:42,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:42,289 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:56:42,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:42,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:43,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:44,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:45,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:45,370 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:56:46,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:46,545 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:47,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:48,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:49,045 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:50,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:51,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:51,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:52,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:53,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:54,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:55,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:56,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:56,690 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:56:57,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:57,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:56:57,652 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:56:58,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:56:59,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:00,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:01,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:01,725 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:02,047 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:03,068 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:03,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:03,754 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:57:03,754 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 with policy now +2025-01-17 14:57:03,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:57:03,765 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:03,765 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:03,766 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:04,013 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/yw7m3f6j-media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:04,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:04,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:04,378 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:04,379 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:57:04,674 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hssrhkv3-media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 14:57:05,056 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:05,379 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:06,060 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:06,847 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:07,094 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:08,064 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:09,076 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:09,380 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:10,074 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:11,076 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:11,381 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:12,088 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:12,281 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:57:12,282 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:12,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:12,678 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:13,098 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:13,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:57:13,451 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:13,451 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:13,453 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:14,095 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:14,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:14,382 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:57:15,108 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:15,383 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:16,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:17,120 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:17,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:18,125 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:19,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:19,384 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:20,136 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:21,140 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:22,128 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:22,705 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:23,132 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:24,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:25,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:25,386 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:26,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:27,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:27,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:27,653 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:27,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:28,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:29,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:30,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:31,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:32,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:33,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:33,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:34,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:35,114 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:36,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:37,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:37,391 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:38,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:39,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:39,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:40,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:41,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:42,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:42,282 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:57:42,653 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:42,654 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:43,115 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:44,116 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:44,758 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:45,116 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:46,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:47,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:48,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:49,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:49,396 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:50,117 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:50,313 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:51,150 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:51,397 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:51,487 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:57:51,487 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 with policy now +2025-01-17 14:57:51,493 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:57:51,493 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:57:51,493 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:57:51,494 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:57:51,748 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/hqx05krv-media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:52,133 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:52,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:57:52,707 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/u4scvtze-media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 14:57:53,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:53,398 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:54,152 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:55,193 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:56,150 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:56,193 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:57:57,159 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:57,400 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:57:57,683 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:57:57,684 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:57:58,146 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:59,154 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:57:59,401 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:00,145 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:01,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:01,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:02,149 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:02,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:02,514 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:02,514 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:02,514 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:03,164 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:03,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:03,402 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:04,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:05,148 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:05,403 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:06,161 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:07,152 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:07,205 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:07,404 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:08,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:09,176 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:10,169 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:11,169 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:12,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:12,282 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:58:12,283 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:12,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:12,672 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:13,179 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:14,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:15,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:15,407 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:16,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:17,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:17,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:18,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:19,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:20,171 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:21,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:22,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:22,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:23,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:24,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:25,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:26,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:27,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:27,665 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:27,666 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:27,813 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:28,172 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:29,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:30,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:31,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:32,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:33,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:33,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:34,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:35,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:36,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:37,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:38,173 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:38,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:39,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:40,174 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:41,181 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:41,418 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:42,179 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:42,271 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:58:42,271 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 with policy now +2025-01-17 14:58:42,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:58:42,285 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:58:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:42,285 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:42,286 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:42,418 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:42,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:42,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:58:42,540 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/x4gcj7og-media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:42,672 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:42,672 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:42,812 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/alyp4d5r-media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 14:58:43,204 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:43,419 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:43,783 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:44,212 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:45,203 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:45,420 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:46,242 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:47,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:47,420 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:48,220 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:49,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:49,240 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:50,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:50,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:50,576 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:58:50,576 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:58:50,581 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:58:50,695 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/jbot3eff-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:51,233 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:51,422 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:58:52,269 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:53,263 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:53,423 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:54,236 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:54,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:58:55,259 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:55,424 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:58:56,261 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:57,238 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:57,686 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:58:57,686 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:58:58,235 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:59,262 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:58:59,844 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:00,288 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:01,258 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:02,265 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:03,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:03,427 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:04,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:05,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:05,309 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:05,428 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:06,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:07,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:08,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:09,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:10,246 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:10,752 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:11,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:12,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:12,283 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:59:12,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:12,673 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:13,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:14,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:15,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:15,775 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:16,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:17,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:18,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:19,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:20,247 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:21,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:21,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:22,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:23,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:24,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:25,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:26,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:27,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:27,673 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:27,674 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:27,795 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:28,248 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:29,249 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:30,297 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:31,274 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:31,716 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 14:59:31,716 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 with policy now +2025-01-17 14:59:31,720 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 14:59:31,721 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:59:31,721 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:59:31,721 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:59:31,998 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/1ah9u18q-media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,281 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:32,440 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:59:32,440 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:32,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 14:59:32,753 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/9n8n3cul-media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 14:59:32,807 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:33,287 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:33,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:34,290 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:34,441 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:35,311 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:36,295 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:37,298 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:38,296 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:38,443 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:38,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:39,300 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:40,297 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:40,444 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:41,313 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:42,284 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 14:59:42,329 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:42,444 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:59:42,641 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 14:59:42,641 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 14:59:42,642 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 14:59:42,697 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:42,697 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:43,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:43,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 14:59:43,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 14:59:44,322 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:44,445 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:44,477 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:45,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:46,360 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:46,446 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:47,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:48,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:48,447 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:49,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:49,797 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:50,325 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:50,448 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 14:59:51,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:52,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:53,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:54,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:55,326 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:55,765 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 14:59:56,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:57,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:57,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 14:59:57,679 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 14:59:58,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 14:59:59,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:00,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:00,800 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:01,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:02,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:03,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:04,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:05,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:06,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:06,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:07,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:08,327 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:09,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:10,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:11,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:11,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:12,284 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:00:12,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:12,679 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:12,680 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:13,328 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:14,345 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:15,348 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:15,621 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:00:15,621 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 with policy now +2025-01-17 15:00:15,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:00:15,644 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:00:15,644 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:00:15,644 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:00:15,886 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/553ul7rh-media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:16,345 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:16,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:00:16,753 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/51414zxw-media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:00:17,347 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:17,459 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:17,734 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:18,350 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:18,460 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:19,350 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:20,353 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:21,376 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:22,363 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:22,461 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:22,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:23,392 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:23,688 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:00:23,688 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:00:23,693 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:00:24,391 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:24,462 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:00:24,462 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:24,463 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:00:25,425 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:26,400 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:26,463 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:27,416 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:27,694 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:27,694 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:27,846 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:28,423 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:29,421 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:30,419 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:30,465 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:31,421 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:32,436 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:33,463 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:33,814 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:34,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:35,466 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:36,469 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:37,486 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:38,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:38,982 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:39,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:40,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:40,469 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:00:41,456 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:00:42,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:42,684 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:42,684 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:43,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:44,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:44,964 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:45,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:46,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:47,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:48,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:49,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:50,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:50,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:51,457 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:52,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:53,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:54,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:55,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:55,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:00:56,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:57,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:57,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:00:57,692 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:00:58,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:00:59,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:00,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:00,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:01,458 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:02,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:03,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:04,459 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:05,480 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:06,046 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:06,213 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:01:06,213 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 with policy now +2025-01-17 15:01:06,235 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:01:06,236 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:06,236 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:06,236 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:06,473 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:06,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:06,481 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:01:06,483 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/5skhsm87-media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:06,756 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/mctt4p8j-media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:01:07,480 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:07,493 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:08,481 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:08,501 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:09,583 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:10,550 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:11,553 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:11,981 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:12,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:01:12,483 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:12,546 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:12,734 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:12,735 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:13,530 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:14,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:01:14,539 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:14,755 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:14,755 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:14,755 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:14,780 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g83t1x47-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:01:15,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:15,577 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:16,484 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:16,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:17,245 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:17,606 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:18,485 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:18,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:19,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:20,486 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:20,559 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:21,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:22,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:22,762 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:23,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:24,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:25,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:26,489 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:26,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:27,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:27,691 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:27,691 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:27,812 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:28,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:29,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:30,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:31,560 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:32,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:32,824 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:33,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:34,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:35,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:36,561 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:37,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:37,824 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:38,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:39,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:40,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:41,562 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:42,285 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:01:42,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:42,692 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:42,692 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:42,838 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:43,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:44,563 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:45,576 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:46,498 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:46,583 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:47,580 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:47,767 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:01:47,767 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 with policy now +2025-01-17 15:01:47,793 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:01:47,793 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:01:47,793 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:01:47,793 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:01:47,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:48,035 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/lqhk7obw-media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:48,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:01:48,624 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:48,794 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/3avpp5qc-media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:01:49,499 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:49,612 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:50,500 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:50,595 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:51,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:52,623 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:52,897 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:53,618 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:54,501 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:54,599 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:55,639 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:56,502 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:01:56,641 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:57,615 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:57,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:01:57,756 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:01:57,957 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:01:58,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:01:59,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:00,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:00,811 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:00,811 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:00,812 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:01,504 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:01,505 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:02:01,652 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:02,505 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:02,626 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:03,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:03,883 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:04,506 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:04,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:05,627 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:06,507 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:06,643 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:07,671 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:08,508 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:08,662 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:09,681 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:09,873 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:10,655 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:11,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:12,286 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:02:12,509 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:12,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:12,712 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:12,713 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:13,645 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:14,510 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:14,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:15,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:15,863 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:16,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:17,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:18,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:19,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:20,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:21,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:21,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:22,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:23,646 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:24,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:25,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:26,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:26,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:27,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:27,713 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:27,713 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:28,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:29,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:30,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:31,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:31,861 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:32,647 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:33,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:34,648 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:35,651 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:36,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:37,650 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:37,860 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:38,657 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:39,426 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:02:39,426 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 with policy now +2025-01-17 15:02:39,447 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:02:39,448 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:39,448 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:39,448 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:39,521 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:39,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:39,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:02:39,659 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:39,699 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qxfe0zj5-media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:39,976 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/sxl2cva2-media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:02:40,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:40,675 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:41,522 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:41,703 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:42,286 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:02:42,523 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:42,682 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:42,772 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:42,773 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:42,912 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:43,711 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:44,524 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:44,687 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:45,729 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:46,525 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:46,678 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:47,685 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:47,868 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:02:47,869 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:02:47,869 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:02:47,937 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:48,526 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:02:48,749 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:49,740 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:50,527 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:50,715 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:51,714 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:52,719 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:53,763 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:53,909 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:02:54,528 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:02:54,730 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:55,764 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:56,741 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:57,735 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:57,764 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:02:57,765 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:02:58,742 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:59,761 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:02:59,904 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:00,766 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:01,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:02,793 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:03,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:04,533 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:04,753 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:05,151 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:05,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:06,535 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:06,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:07,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:08,536 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:08,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:09,754 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:10,537 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:10,745 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:10,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:11,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:12,287 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:03:12,538 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:12,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:12,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:12,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:13,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:14,539 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:14,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:15,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:15,985 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:16,540 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:16,755 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:17,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:18,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:19,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:20,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:21,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:21,891 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:22,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:23,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:24,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:25,756 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:26,544 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:26,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:27,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:27,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:27,789 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:27,917 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:28,545 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:28,757 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:29,757 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:30,798 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:31,602 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:31,602 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 with policy now +2025-01-17 15:03:31,640 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:31,640 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:31,640 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:31,641 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:31,780 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:31,872 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/x46k3m8c-media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:32,547 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:32,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval +2025-01-17 15:03:32,799 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:32,871 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/jl6hd4vq-media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:03:33,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:33,791 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:33,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:34,548 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:34,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:35,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:36,839 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:37,840 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:38,550 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:38,831 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:38,913 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:39,550 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:39,834 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:39,932 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:39,932 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:39,933 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:40,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:40,831 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:40,921 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 15:03:40,921 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 15:03:41,139 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:41,551 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval +2025-01-17 15:03:41,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table +2025-01-17 15:03:41,558 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 15:03:41,570 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 15:03:41,635 INFO wandb-upload_2:2517552 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf7i9yzmj +2025-01-17 15:03:41,793 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/thkplcdy-media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:41,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:41,855 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/qtky6ri1-events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:03:42,076 INFO SenderThread:2517552 [sender.py:send_request_log_artifact():1525] logged artifact run-up07mbix-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEzMjcwMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU2Mjc2NA==', 'latestArtifact': None}} +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:42,076 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json with policy now +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:42,076 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,077 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,077 DEBUG SenderThread:2517552 [sender.py:send():379] send: telemetry +2025-01-17 15:03:42,077 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,081 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,081 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,081 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,082 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,082 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,082 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,083 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,083 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,083 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,083 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,084 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,084 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,084 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,085 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,085 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,085 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,085 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,086 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,086 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,086 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,087 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,087 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 15:03:42,343 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/ulxn4kk7-media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:03:42,473 INFO wandb-upload_3:2517552 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppfo4cq0c +2025-01-17 15:03:42,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:42,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval +2025-01-17 15:03:42,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:42,802 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:42,904 INFO SenderThread:2517552 [sender.py:send_request_log_artifact():1525] logged artifact run-up07mbix-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTEzMjcyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTU2Mjc2Nw==', 'latestArtifact': None}} +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send():379] send: files +2025-01-17 15:03:42,904 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json with policy now +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:03:42,904 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:03:42,905 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:03:42,905 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:03:42,905 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:43,158 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/6l91m2mj-media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:43,428 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /tmp/tmp7sbm32f9wandb/g6lnbs6v-media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:03:43,552 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:03:43,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:43,972 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:44,553 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:03:44,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:45,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:46,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:47,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:48,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:49,803 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:49,894 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:50,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:51,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:52,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:53,804 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:54,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:54,894 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:03:55,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:56,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:57,743 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 15:03:57,743 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: stop_status +2025-01-17 15:03:57,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:58,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:59,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:03:59,898 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:00,561 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 15:04:00,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:01,805 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:02,405 DEBUG SenderThread:2517552 [sender.py:send():379] send: exit +2025-01-17 15:04:02,405 INFO SenderThread:2517552 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_exit():588] handling runtime: 980 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_exit():594] send defer +2025-01-17 15:04:02,406 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,406 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 15:04:02,406 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 15:04:02,406 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 1 +2025-01-17 15:04:02,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 15:04:02,407 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,407 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 15:04:02,407 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 2 +2025-01-17 15:04:02,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 15:04:02,407 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 15:04:02,407 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 15:04:02,408 DEBUG SystemMonitor:2517552 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 15:04:02,408 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 15:04:02,458 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 15:04:02,459 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 15:04:02,459 INFO HandlerThread:2517552 [interfaces.py:finish():200] Joined network monitor +2025-01-17 15:04:02,459 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,459 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 15:04:02,459 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 3 +2025-01-17 15:04:02,459 DEBUG SenderThread:2517552 [sender.py:send():379] send: stats +2025-01-17 15:04:02,460 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,460 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 15:04:02,460 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:02,460 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 15:04:02,460 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 4 +2025-01-17 15:04:02,460 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:02,460 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 15:04:02,562 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:04,564 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:08,941 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,942 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 15:04:08,942 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 5 +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:08,942 DEBUG SenderThread:2517552 [sender.py:send():379] send: history +2025-01-17 15:04:08,943 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: summary_record +2025-01-17 15:04:08,943 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,943 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 15:04:08,944 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:08,944 DEBUG SenderThread:2517552 [sender.py:send():379] send: summary +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 15:04:08,948 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 6 +2025-01-17 15:04:08,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,948 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 15:04:08,948 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 15:04:08,948 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 7 +2025-01-17 15:04:08,948 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 15:04:08,949 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:08,949 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 15:04:08,949 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:08,949 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 15:04:09,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:09,566 INFO Thread-12 :2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:11,093 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 8 +2025-01-17 15:04:11,093 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:11,094 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,094 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 15:04:11,094 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,094 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 15:04:11,094 INFO SenderThread:2517552 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 15:04:11,097 INFO SenderThread:2517552 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 15:04:11,099 INFO SenderThread:2517552 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 15:04:11,113 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 9 +2025-01-17 15:04:11,113 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,114 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 15:04:11,114 DEBUG SenderThread:2517552 [sender.py:send():379] send: artifact +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTEwNDY1OA==', 'versionIndex': 442}}} +2025-01-17 15:04:11,314 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 15:04:11,314 INFO SenderThread:2517552 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 15:04:11,407 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:11,567 INFO SenderThread:2517552 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:11,567 INFO SenderThread:2517552 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml config.yaml +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json wandb-summary.json +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt requirements.txt +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-metadata.json wandb-metadata.json +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 events.out.tfevents.1737143262.leonmkim-ROG-Strix-G15CS-G15CS.2517388.0 +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log output.log +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/diff.patch diff.patch +2025-01-17 15:04:11,568 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 15:04:11,574 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 media/videos/eval/7_eval_7_baeae50ab773d27ae44b.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 media/videos/eval/10_eval_10_268ff9e2e53b41188aa0.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 media/videos/eval/16_eval_16_d19b3abe759fefca6827.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 media/videos/eval/0_eval_0_1819f930b352d4eea54c.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 media/videos/eval/6_eval_6_a92f6e4d8653d4439383.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 media/videos/eval/14_eval_14_bfd6e56a3506bb346792.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 media/videos/eval/17_eval_17_9039e216f093cde1d3f4.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 media/videos/eval/12_eval_12_79b7fe799da0fe5e29e5.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 media/videos/eval/1_eval_1_093e197b5aca9677627b.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 media/videos/eval/15_eval_15_9e02939fb08b9aeb13e6.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 media/videos/eval/4_eval_4_916d91435e19178c30f6.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 media/videos/eval/5_eval_5_abbb012eb02e7137181f.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 media/videos/eval/3_eval_3_ccfd7c2808a5399343db.mp4 +2025-01-17 15:04:11,575 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 media/videos/eval/13_eval_13_72586beaede9059dffaa.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 media/videos/eval/8_eval_8_f6383c0f370fe43d88dd.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 media/videos/eval/18_eval_18_d73f4283df1c3c229b04.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 media/videos/eval/9_eval_9_a200f16787e6024f6581.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 media/videos/eval/19_eval_19_1265d28d813382198c39.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 media/videos/eval/11_eval_11_30dc16b787631229f366.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 media/videos/eval/2_eval_2_ee6ae32e8cb4b17611f2.mp4 +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json media/table/eval/total_success_rate_ci_21_37909e05a422137136b7.table.json +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json media/table/eval/success_rate_ci_20_fc42d684313cd17be1f8.table.json +2025-01-17 15:04:11,576 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 10 +2025-01-17 15:04:11,576 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:11,576 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:11,577 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 15:04:11,577 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:11,577 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 15:04:11,577 INFO SenderThread:2517552 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:04:11,763 INFO wandb-upload_2:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/wandb-summary.json +2025-01-17 15:04:11,792 INFO wandb-upload_3:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/requirements.txt +2025-01-17 15:04:11,805 INFO wandb-upload_0:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/config.yaml +2025-01-17 15:04:11,814 INFO wandb-upload_1:2517552 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/files/output.log +2025-01-17 15:04:12,015 INFO Thread-11 (_thread_body):2517552 [sender.py:transition_state():614] send defer: 11 +2025-01-17 15:04:12,015 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,015 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 15:04:12,015 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,015 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 15:04:12,015 INFO SenderThread:2517552 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 12 +2025-01-17 15:04:12,019 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,019 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 15:04:12,019 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 15:04:12,019 INFO SenderThread:2517552 [file_stream.py:finish():601] file stream finish called +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [file_stream.py:finish():605] file stream finish is done +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 13 +2025-01-17 15:04:12,217 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,217 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 15:04:12,217 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 15:04:12,217 INFO SenderThread:2517552 [sender.py:transition_state():614] send defer: 14 +2025-01-17 15:04:12,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: defer +2025-01-17 15:04:12,218 INFO HandlerThread:2517552 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send():379] send: final +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send():379] send: footer +2025-01-17 15:04:12,218 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: defer +2025-01-17 15:04:12,218 INFO SenderThread:2517552 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 15:04:12,218 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:12,219 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 15:04:12,219 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: server_info +2025-01-17 15:04:12,219 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 15:04:12,221 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 15:04:12,222 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 15:04:12,282 DEBUG SenderThread:2517552 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 15:04:12,282 INFO MainThread:2517552 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 15:04:12,283 INFO MainThread:2517552 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 15:04:12,283 INFO MainThread:2517552 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 15:04:12,283 DEBUG HandlerThread:2517552 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 15:04:12,283 INFO HandlerThread:2517552 [handler.py:finish():882] shutting down handler +2025-01-17 15:04:13,220 INFO WriterThread:2517552 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [sender.py:finish():1615] shutting down sender +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 15:04:13,282 INFO SenderThread:2517552 [file_pusher.py:join():175] waiting for file pusher diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug.log b/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..679e797b9970446285c36af14b01b59cafb466a3 --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Configure stats pid to 2517388 +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 14:47:41,914 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/settings +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug.log +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/wandb/run-20250117_144741-up07mbix/logs/debug-internal.log +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():569] calling init triggers +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix'} +2025-01-17 14:47:41,915 INFO MainThread:2517388 [wandb_init.py:init():619] starting backend +2025-01-17 14:47:41,916 INFO MainThread:2517388 [wandb_init.py:init():623] setting up manager +2025-01-17 14:47:41,920 INFO MainThread:2517388 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 14:47:41,921 INFO MainThread:2517388 [wandb_init.py:init():631] backend started and connected +2025-01-17 14:47:41,933 INFO MainThread:2517388 [wandb_init.py:init():720] updated telemetry +2025-01-17 14:47:41,939 INFO MainThread:2517388 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 14:47:42,181 INFO MainThread:2517388 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 14:47:42,220 INFO MainThread:2517388 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 14:47:42,220 INFO MainThread:2517388 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 14:47:42,627 INFO MainThread:2517388 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 14:47:42,628 INFO MainThread:2517388 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 14:47:42,628 INFO MainThread:2517388 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/up07mbix/tb, True +2025-01-17 14:47:48,084 INFO MainThread:2517388 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 15:04:13,286 WARNING MsgRouterThr:2517388 [router.py:message_loop():77] message_loop has been closed diff --git a/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb b/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9a8479a5fa022c5b68ba95328200923eae171d0d --- /dev/null +++ b/up07mbix/wandb/run-20250117_144741-up07mbix/run-up07mbix.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ab48dc5c506f83b4c6a4526deadf8d8a930946481b4937ebccf12e016cee931 +size 241713 diff --git a/vfeb54wi/episode_rosbags/aligned_depth_to_color_K.npy b/vfeb54wi/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vfeb54wi/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vfeb54wi/episode_rosbags/cam_tf_world.npy b/vfeb54wi/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/vfeb54wi/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/vfeb54wi/episode_rosbags/color_K.npy b/vfeb54wi/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/vfeb54wi/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/vfeb54wi/episode_rosbags/depth_K.npy b/vfeb54wi/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/vfeb54wi/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/vfeb54wi/episode_rosbags/episode_10_2025-01-15-21-34-09.bag b/vfeb54wi/episode_rosbags/episode_10_2025-01-15-21-34-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..002519614c39e0fdb70b7eba859158dbb4e37475 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_10_2025-01-15-21-34-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c68e5cdffbf220c7319ad0714adf237f9226011af6b2074bbb70a61208ff8f55 +size 1405554942 diff --git a/vfeb54wi/episode_rosbags/episode_16_2025-01-15-21-38-58.bag b/vfeb54wi/episode_rosbags/episode_16_2025-01-15-21-38-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..894db149668f7f6a337fcbc9a776a2e77e57ca2f --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_16_2025-01-15-21-38-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8f2b1888fdcb7c0c1af65bf2a381e3f50c39fac1c8d90e57478c3a035a644d +size 1413128387 diff --git a/vfeb54wi/episode_rosbags/episode_3_2025-01-15-21-29-12.bag b/vfeb54wi/episode_rosbags/episode_3_2025-01-15-21-29-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..d3678588d6109b8aeae4938210bc013c6204a64e --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_3_2025-01-15-21-29-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe01e073a1f10edaadfa3e57f260aa03fb1a43908100eaa0478e616faebcb535 +size 1411246935 diff --git a/vfeb54wi/episode_rosbags/episode_4_2025-01-15-21-29-58.bag b/vfeb54wi/episode_rosbags/episode_4_2025-01-15-21-29-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..88c8e0989622f9dd6eab5a63885537fe60db89aa --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_4_2025-01-15-21-29-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d227977af4702377bb0d90b3141fac9ae7ecfd19bbb9529a120932d8d97a6a8 +size 1407315840 diff --git a/vfeb54wi/episode_rosbags/episode_5_2025-01-15-21-30-43.bag b/vfeb54wi/episode_rosbags/episode_5_2025-01-15-21-30-43.bag new file mode 100644 index 0000000000000000000000000000000000000000..1da46726f131b6cd6d2c9bf1fd8f2ecc5c44d06a --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_5_2025-01-15-21-30-43.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf93b3e0d072589dbe490aec04de02c9d1decab8ff1f9ad34b378cc708a7d40 +size 828166759 diff --git a/vfeb54wi/episode_rosbags/episode_6_2025-01-15-21-31-20.bag b/vfeb54wi/episode_rosbags/episode_6_2025-01-15-21-31-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..32a72f4509baab41ab4363934a81d8cec409aca0 --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_6_2025-01-15-21-31-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca46da1430120e6e361cd5185d2230cf21cf8b9b2db345dd79b8e9800e78d606 +size 770381672 diff --git a/vfeb54wi/episode_rosbags/episode_7_2025-01-15-21-31-54.bag b/vfeb54wi/episode_rosbags/episode_7_2025-01-15-21-31-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..4ce15ffeab2b1d6f2a16e92eeca396580565db1d --- /dev/null +++ b/vfeb54wi/episode_rosbags/episode_7_2025-01-15-21-31-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6654c049295e78948a7dc0c72268822dceddc9e4830531c732253d02ff275ac +size 743185363 diff --git a/vfeb54wi/eval_robot.log b/vfeb54wi/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0e24148efb26aab5e3c716911dd1689615a1bcbf --- /dev/null +++ b/vfeb54wi/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-15 21:25:06,816][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-15 21:25:06,819][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/vfeb54wi/eval_video/12_eval.mp4 b/vfeb54wi/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a92778174d96e5ad8d41d9cc584fbeab00538049 --- /dev/null +++ b/vfeb54wi/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7f16c34c55c022ec9cf4d75084ba2f749e4b014ef3b58dd31fb9421f10d545c +size 1176474 diff --git a/vfeb54wi/eval_video/19_eval.mp4 b/vfeb54wi/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41916924492b3fef800d9cd2176bee811bdd2011 --- /dev/null +++ b/vfeb54wi/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c347402d83b94d0d76b5e3a7d52f138909cd08d1025f74ebc5b9ca9b69ed8972 +size 1188193 diff --git a/vfeb54wi/tb/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 b/vfeb54wi/tb/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 new file mode 100644 index 0000000000000000000000000000000000000000..bde9e391c6b836e0908fa64823983bdec7602f82 --- /dev/null +++ b/vfeb54wi/tb/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f57a5d35d12e1a2c725ab74799065dbcf88bb24db723c3e8f4d6b8b9e55edf0e +size 22784 diff --git a/vfeb54wi/wandb/debug-internal.log b/vfeb54wi/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2e2530901a748dc6ba4a612e70c6bdca3e99947 --- /dev/null +++ b/vfeb54wi/wandb/debug-internal.log @@ -0,0 +1,2245 @@ +2025-01-15 21:25:12,918 INFO StreamThr :2467459 [internal.py:wandb_internal():85] W&B internal server running at pid: 2467459, started at: 2025-01-15 21:25:12.918214 +2025-01-15 21:25:12,919 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status +2025-01-15 21:25:12,929 INFO WriterThread:2467459 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb +2025-01-15 21:25:12,929 DEBUG SenderThread:2467459 [sender.py:send():379] send: header +2025-01-15 21:25:12,938 DEBUG SenderThread:2467459 [sender.py:send():379] send: run +2025-01-15 21:25:13,150 INFO SenderThread:2467459 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files +2025-01-15 21:25:13,150 INFO SenderThread:2467459 [sender.py:_start_run_threads():1188] run started: vfeb54wi with start time 1736994312.918604 +2025-01-15 21:25:13,164 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: check_version +2025-01-15 21:25:13,164 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: check_version +2025-01-15 21:25:13,218 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: run_start +2025-01-15 21:25:13,223 DEBUG HandlerThread:2467459 [system_info.py:__init__():26] System info init +2025-01-15 21:25:13,223 DEBUG HandlerThread:2467459 [system_info.py:__init__():41] System info init done +2025-01-15 21:25:13,223 INFO HandlerThread:2467459 [system_monitor.py:start():194] Starting system monitor +2025-01-15 21:25:13,223 INFO SystemMonitor:2467459 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-15 21:25:13,223 INFO HandlerThread:2467459 [system_monitor.py:probe():214] Collecting system info +2025-01-15 21:25:13,224 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started cpu monitoring +2025-01-15 21:25:13,224 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started disk monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started gpu monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started memory monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started network monitoring +2025-01-15 21:25:13,236 DEBUG HandlerThread:2467459 [system_info.py:probe():152] Probing system +2025-01-15 21:25:13,238 DEBUG HandlerThread:2467459 [system_info.py:_probe_git():137] Probing git +2025-01-15 21:25:13,242 DEBUG HandlerThread:2467459 [system_info.py:_probe_git():145] Probing git done +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_info.py:probe():200] Probing system done +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-16T02:25:13.236976', 'startedAt': '2025-01-16T02:25:12.910503', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.49975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 302.54129791259766}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-15 21:25:13,243 INFO HandlerThread:2467459 [system_monitor.py:probe():224] Finished collecting system info +2025-01-15 21:25:13,243 INFO HandlerThread:2467459 [system_monitor.py:probe():227] Publishing system info +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_info.py:_save_code():44] Saving code +2025-01-15 21:25:13,244 DEBUG HandlerThread:2467459 [system_info.py:_save_code():67] Saving code done +2025-01-15 21:25:13,244 DEBUG HandlerThread:2467459 [system_info.py:_save_patches():84] Saving git patches +2025-01-15 21:25:13,340 DEBUG HandlerThread:2467459 [system_info.py:_save_patches():126] Saving git patches done +2025-01-15 21:25:13,341 INFO HandlerThread:2467459 [system_monitor.py:probe():229] Finished publishing system info +2025-01-15 21:25:13,344 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-15 21:25:13,345 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-15 21:25:13,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: python_packages +2025-01-15 21:25:13,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: python_packages +2025-01-15 21:25:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:13,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:13,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:13,599 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fdj573ij-wandb-metadata.json +2025-01-15 21:25:13,609 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/jmoc2a32-code/FISH/eval_robot.py +2025-01-15 21:25:13,620 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/346yqiic-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:25:13,623 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/4asr6hxf-diff.patch +2025-01-15 21:25:13,629 INFO HandlerThread:2467459 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/tb" + save: true +} +control { + relay_id: "83dc957e0c7848d8aed4259b6fd3c7e4" +} +_info { + stream_id: "vfeb54wi" +} + +2025-01-15 21:25:13,629 DEBUG HandlerThread:2467459 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-15 21:25:13,666 DEBUG SenderThread:2467459 [sender.py:send():379] send: telemetry +2025-01-15 21:25:13,697 DEBUG SenderThread:2467459 [sender.py:send():379] send: tbrecord +2025-01-15 21:25:13,698 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:25:13,698 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 with policy live +2025-01-15 21:25:13,891 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/6y9bdqf3-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:25:14,151 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code +2025-01-15 21:25:14,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:15,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:16,151 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:16,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:17,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:18,431 DEBUG SenderThread:2467459 [sender.py:send():379] send: config +2025-01-15 21:25:18,431 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:18,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:19,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:20,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:20,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:21,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:22,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:24,153 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:24,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:25,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:26,154 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:26,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:27,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:28,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:28,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:28,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:28,727 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:29,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:30,155 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:30,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:31,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:32,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:33,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:34,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:34,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:35,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:36,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:37,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:38,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:39,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:39,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:40,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:41,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:42,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:43,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:43,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:44,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:44,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:45,159 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:25:45,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:46,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:47,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:48,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:49,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:50,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:50,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:51,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:52,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:53,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:54,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:55,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:56,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:56,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:57,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:58,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:58,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:58,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:59,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:00,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:01,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:01,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:02,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:03,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:04,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:05,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:06,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:07,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:07,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:08,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:09,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:10,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:11,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:12,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:12,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:13,225 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-15 21:26:13,226 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:26:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:13,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:13,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:14,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:15,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:16,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:17,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:17,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:18,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:19,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:20,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:21,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:22,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:23,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:23,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:24,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:25,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:26,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:27,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:28,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:28,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:28,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:28,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:29,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:30,172 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:30,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:31,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:32,172 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:32,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:33,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:34,173 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:34,191 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:34,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:35,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:36,173 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:36,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:37,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:38,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:39,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:39,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:40,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:41,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:42,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:43,227 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:26:43,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:43,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:44,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:44,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:45,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:46,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:47,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:48,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:49,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:50,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:50,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:51,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:52,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:53,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:54,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:55,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:55,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:56,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:57,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:58,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:58,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:58,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:58,973 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:26:58,973 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 with policy now +2025-01-15 21:26:58,973 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:26:58,974 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:26:58,974 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:26:58,976 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos +2025-01-15 21:26:59,209 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/kjtcrf0m-media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,476 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/s0tmh9b8-media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:00,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:00,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:00,980 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:01,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:02,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:03,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:04,182 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:04,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:05,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:06,182 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:06,333 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:06,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:07,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:08,183 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:08,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:09,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:10,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:11,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:11,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:12,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:13,227 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:27:13,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:13,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:13,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:14,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:15,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:16,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:16,736 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:17,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:18,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:19,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:20,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:21,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:22,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:22,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:23,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:24,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:25,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:26,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:27,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:27,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:28,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:28,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:28,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:29,190 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:27:29,450 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/twg1wjqp-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:27:29,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:30,191 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:30,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:31,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:32,192 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:32,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:32,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:33,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:34,192 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:34,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:35,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:36,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:37,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:38,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:38,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:39,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:40,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:41,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:42,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:43,228 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:27:43,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:43,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:43,680 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:44,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:45,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:46,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:47,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:48,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:49,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:49,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:50,198 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:50,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:51,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:52,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:53,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:54,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:54,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:55,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:56,200 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:56,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:57,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:58,200 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:58,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:58,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:58,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:59,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:59,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:00,201 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:00,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:01,080 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:28:01,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:28:01,080 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 with policy now +2025-01-15 21:28:01,081 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:01,081 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:01,094 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:28:01,306 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/rfbzb3bw-media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,573 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/0x6crfkr-media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:02,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:02,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:03,611 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:04,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:04,702 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:05,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:06,204 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:06,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:07,611 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:08,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:09,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:10,205 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:10,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:10,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:11,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:12,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:13,228 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:28:13,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:13,586 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:13,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:14,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:15,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:15,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:16,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:17,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:18,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:19,614 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:20,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:21,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:21,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:22,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:23,209 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:28:23,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:24,141 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:24,141 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:24,142 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:24,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:24,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:24,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:25,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:26,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:26,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:26,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:27,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:28,211 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:28,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:28,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:28,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:29,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:30,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:31,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:32,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:32,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:33,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:34,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:35,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:36,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:37,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:38,501 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:38,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:39,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:40,215 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:40,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:41,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:42,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:43,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:28:43,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:43,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:43,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:43,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:44,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:45,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:46,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:47,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:48,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:49,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:49,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:50,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:51,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:52,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:53,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:54,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:54,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:55,273 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:28:55,273 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 with policy now +2025-01-15 21:28:55,276 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:28:55,277 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:55,277 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:55,283 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:55,490 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/h8kv98eg-media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:55,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:56,220 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:28:56,513 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/nframv28-media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:56,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:57,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:57,627 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:58,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:58,588 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:58,633 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:59,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:59,961 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:00,222 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:00,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:01,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:02,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:03,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:04,224 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:04,632 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:05,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:05,630 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:06,628 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:07,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:08,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:08,978 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:08,978 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:08,978 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:09,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:09,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:09,488 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/t9k3stc7-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:09,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:10,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:10,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:10,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:11,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:12,228 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:12,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:13,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:29:13,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:13,586 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:13,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:14,229 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:14,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:15,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:16,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:16,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:17,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:18,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:19,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:20,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:21,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:22,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:22,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:23,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:24,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:25,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:26,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:27,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:27,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:28,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:28,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:28,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:29,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:30,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:31,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:32,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:32,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:33,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:34,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:35,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:36,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:37,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:38,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:38,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:39,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:40,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:40,647 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:40,966 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:29:40,966 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 with policy now +2025-01-15 21:29:40,970 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:29:40,971 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:40,971 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:40,971 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:41,207 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qxd14u76-media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:41,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:29:41,494 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/v035hc8i-media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:42,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:42,653 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:43,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:29:43,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:43,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:43,588 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:43,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:43,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:44,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:45,668 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:46,240 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:46,657 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:47,654 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:48,241 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:48,660 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:49,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:49,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:50,660 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:51,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:52,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:53,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:54,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:55,032 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:55,033 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:55,033 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:55,034 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:55,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:55,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:55,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:56,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:56,672 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:57,669 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:58,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:58,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:58,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:59,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:00,245 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:00,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:00,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:01,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:02,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:03,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:04,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:05,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:06,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:06,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:07,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:08,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:09,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:10,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:11,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:11,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:12,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:13,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:30:13,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:13,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:13,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:14,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:15,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:16,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:16,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:17,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:18,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:19,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:20,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:21,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:22,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:22,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:23,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:24,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:25,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:26,242 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:30:26,242 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 with policy now +2025-01-15 21:30:26,245 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:30:26,246 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:30:26,246 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:30:26,246 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:30:26,254 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:30:26,459 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/wdw28quw-media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:26,674 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:26,725 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/a1s92lgd-media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:27,683 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:28,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:28,268 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:28,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:28,589 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:28,684 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:29,682 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:30,684 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:31,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:32,256 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:32,685 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:33,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:33,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:34,257 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:34,686 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:35,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:36,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:37,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:38,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:38,694 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:39,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:40,088 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:30:40,088 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:30:40,090 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:30:40,259 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:30:40,259 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:30:40,509 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/bziwa5lu-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:30:40,701 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:41,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:42,260 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:42,700 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:43,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:30:43,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:43,610 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:43,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:43,694 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:44,260 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:44,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:45,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:46,261 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:46,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:47,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:48,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:49,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:49,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:50,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:51,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:52,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:53,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:54,264 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:54,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:54,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:55,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:56,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:57,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:58,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:58,589 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:58,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:59,669 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:59,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:00,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:01,072 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:31:01,072 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 with policy now +2025-01-15 21:31:01,079 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:31:01,079 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:01,079 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:01,080 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:31:01,278 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ckcq6zx5-media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,538 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/1osnfgsm-media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,696 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:02,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:02,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:03,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:04,268 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:04,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:04,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:05,701 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:06,268 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:06,702 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:07,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:08,269 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:08,707 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:09,708 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:10,270 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:10,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:10,703 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:11,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:12,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:13,231 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:31:13,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:13,602 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:13,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:14,721 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:15,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:15,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:16,727 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:17,135 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:17,135 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:17,135 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:17,272 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:17,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:17,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:18,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:18,722 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:19,735 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:20,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:20,732 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:20,785 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:21,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:22,274 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:22,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:23,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:24,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:25,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:26,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:26,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:27,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:28,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:28,591 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:28,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:29,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:30,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:31,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:31,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:32,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:33,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:34,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:35,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:36,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:37,053 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:37,400 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:31:37,400 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 with policy now +2025-01-15 21:31:37,423 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:31:37,423 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:37,423 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:37,424 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:37,632 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/9oj5qj44-media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:37,735 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:31:38,548 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ll3s455q-media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:38,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:39,731 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:40,281 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:40,734 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:41,739 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:42,281 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:42,740 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:43,041 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:43,231 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:31:43,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:43,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:43,751 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:44,752 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:45,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:46,284 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:46,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:47,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:48,620 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:48,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:49,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:50,769 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:51,179 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:51,179 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:51,180 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:51,285 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:51,286 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:51,539 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/nwvjd0p0-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:51,755 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:52,286 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:52,760 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:53,764 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:54,276 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:54,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:55,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:56,287 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:56,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:57,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:58,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:58,591 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:58,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:59,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:59,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:00,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:01,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:02,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:03,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:04,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:05,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:05,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:06,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:07,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:08,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:09,777 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:10,261 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:32:10,261 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 with policy now +2025-01-15 21:32:10,271 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:32:10,271 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:32:10,271 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:32:10,272 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:32:10,292 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:10,292 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:32:10,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:32:10,473 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/b50a1vd9-media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,735 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/yar3cn8d-media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,771 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:11,305 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:11,774 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:12,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:12,783 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:13,232 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:32:13,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:13,600 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:13,774 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:14,773 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:15,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:16,294 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:16,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:16,799 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:17,786 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:18,295 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:18,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:19,788 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:20,787 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:21,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:22,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:22,789 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:23,791 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:24,817 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:25,795 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:26,793 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:27,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:27,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:28,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:28,607 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:28,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:29,818 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:30,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:31,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:32,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:32,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:33,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:34,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:35,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:36,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:32:36,235 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:32:36,235 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:32:36,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:32:36,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:32:36,818 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:37,816 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:38,257 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:38,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:38,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:39,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:40,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:41,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:42,304 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:42,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:43,232 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:32:43,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:43,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:43,629 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:43,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:44,305 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:44,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:45,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:46,306 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:46,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:47,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:48,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:48,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:49,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:50,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:51,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:52,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:53,620 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:53,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:54,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:55,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:56,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:57,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:58,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:58,593 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:58,752 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:58,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:59,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:00,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:01,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:02,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:03,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:04,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:04,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:05,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:06,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:07,824 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:08,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:08,740 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:33:08,740 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 with policy now +2025-01-15 21:33:08,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:33:08,758 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:08,758 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:08,759 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:08,823 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:09,052 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qbg4vmhh-media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:33:09,612 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/lygq9vxg-media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,803 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:09,829 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:10,315 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:10,826 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:11,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:12,843 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:13,233 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:33:13,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:13,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:13,827 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:14,316 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:14,835 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:15,389 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:15,825 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:16,317 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:16,828 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:17,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:18,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:19,834 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:20,640 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:20,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:21,844 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:22,291 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:22,291 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:22,292 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:22,571 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/h9u60pi3-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:33:22,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:23,842 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:24,320 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:24,862 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:25,686 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:25,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:26,321 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:26,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:27,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:28,322 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:28,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:28,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:28,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:29,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:30,765 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:30,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:31,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:32,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:33,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:34,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:35,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:36,113 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:36,325 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:36,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:37,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:38,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:39,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:40,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:41,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:41,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:42,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:43,233 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:33:43,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:43,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:43,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:44,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:45,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:46,689 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:46,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:47,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:48,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:49,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:50,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:51,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:52,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:52,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:53,853 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:54,313 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:33:54,313 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 with policy now +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:33:54,335 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:33:54,336 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:54,336 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:54,336 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:54,544 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/mdztdjn2-media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,811 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/iswk2gaz-media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,848 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:55,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:55,854 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:56,333 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:56,857 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:57,871 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:57,999 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:58,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:58,612 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:58,858 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:59,869 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:00,334 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:00,877 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:01,878 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:02,335 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:02,880 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:03,025 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:03,887 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:04,889 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:05,880 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:06,336 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:06,337 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:06,346 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:06,346 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:06,346 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:06,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:07,337 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:07,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:08,338 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:08,679 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:08,913 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:09,899 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:10,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:11,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:12,341 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:12,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:13,234 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:34:13,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:13,597 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:13,741 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:13,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:14,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:15,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:16,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:17,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:18,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:19,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:19,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:20,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:21,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:22,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:23,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:24,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:24,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:25,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:26,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:27,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:28,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:28,597 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:28,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:29,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:29,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:30,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:31,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:32,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:33,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:34,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:35,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:35,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:36,895 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:37,930 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:38,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:38,412 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:34:38,412 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 with policy now +2025-01-15 21:34:38,432 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:34:38,432 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:38,432 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:38,433 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:38,634 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/lf60jaaj-media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:38,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:39,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:39,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:39,352 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:34:39,630 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qhot98oe-media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:39,907 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:40,352 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:40,656 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:40,933 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:41,941 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:42,944 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:43,234 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:34:43,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:43,608 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:43,943 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:44,353 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:44,938 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:45,956 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:46,293 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:46,354 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:46,946 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:47,944 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:48,942 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:49,942 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:50,355 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:50,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:50,401 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:50,401 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:50,401 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:50,605 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ix0sfkor-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:50,954 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:51,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:51,930 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:51,946 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:52,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:52,961 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:53,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:54,965 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:55,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:56,358 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:56,977 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:57,045 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:57,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:58,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:58,621 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:58,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:59,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:00,361 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:00,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:01,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:02,077 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:02,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:03,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:04,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:05,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:06,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:07,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:07,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:08,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:09,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:10,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:11,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:12,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:12,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:13,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:35:13,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:13,598 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:13,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:14,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:15,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:16,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:17,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:17,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:18,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:19,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:20,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:21,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:22,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:23,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:23,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:24,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:25,971 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:26,371 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:27,026 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:27,203 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:35:27,204 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 with policy now +2025-01-15 21:35:27,216 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:35:27,217 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:35:27,217 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:35:27,217 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:35:27,371 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:35:27,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:27,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:35:27,429 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/kgx5r1rf-media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:27,710 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/d4et035n-media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:28,006 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:28,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:28,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:28,636 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:28,717 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:29,017 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:30,024 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:30,373 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:31,053 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:32,037 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:32,373 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:33,029 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:34,042 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:34,142 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:35,032 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:36,034 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:36,375 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:37,033 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:38,044 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:39,059 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:39,678 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:40,087 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:41,078 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:42,057 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:43,073 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:43,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:35:43,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:35:43,465 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:35:43,465 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:35:43,466 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:35:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:43,625 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:44,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:35:45,091 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:45,653 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:46,095 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:46,379 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:47,106 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:48,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:48,380 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:49,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:50,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:50,381 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:50,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:51,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:52,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:53,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:54,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:55,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:55,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:56,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:57,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:58,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:58,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:58,618 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:59,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:00,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:00,764 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:01,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:02,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:03,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:04,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:05,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:06,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:06,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:07,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:08,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:09,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:10,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:10,389 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:11,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:12,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:12,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:13,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:13,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:36:13,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:13,618 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:14,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:14,390 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:15,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:16,104 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:16,391 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:16,591 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:36:16,591 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 with policy now +2025-01-15 21:36:16,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:36:16,601 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:36:16,601 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:36:16,601 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:36:16,804 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/i49yfhes-media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,102 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:36:17,663 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/xnhcm2ii-media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,674 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:18,122 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:18,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:19,103 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:20,124 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:21,132 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:22,138 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:22,394 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:23,128 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:23,429 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:24,155 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:24,394 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:25,158 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:26,155 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:27,160 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:28,163 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:28,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:28,648 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:28,761 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:29,147 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:30,149 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:31,160 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:32,167 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:32,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:36:32,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:32,527 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:36:32,527 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:36:32,527 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:36:33,183 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:33,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:36:34,179 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:34,399 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:34,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:35,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:36,164 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:36,399 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:37,184 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:38,183 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:38,400 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:39,206 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:40,178 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:40,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:41,194 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:42,188 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:43,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:43,236 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:36:43,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:43,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:44,180 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:44,403 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:45,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:46,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:46,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:47,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:48,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:49,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:50,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:51,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:51,689 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:52,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:53,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:54,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:55,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:56,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:57,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:57,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:58,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:58,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:58,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:59,198 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:00,014 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:37:00,014 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 with policy now +2025-01-15 21:37:00,025 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:37:00,025 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:00,025 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:00,026 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:00,190 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:00,230 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/q2rr96o6-media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:00,410 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:37:00,677 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/rl4f38nx-media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:01,230 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:02,192 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:02,410 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:03,090 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:03,205 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:04,204 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:04,411 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:05,203 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:06,226 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:07,196 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:08,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:08,412 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:08,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:09,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:10,219 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:11,210 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:12,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:13,226 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:13,236 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:37:13,575 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:13,576 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:13,576 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:13,656 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:13,656 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:13,746 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:14,267 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:14,666 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/npemfisv-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:15,228 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:16,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:16,416 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:17,228 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:18,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:18,416 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:19,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:19,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:20,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:20,417 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:21,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:22,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:23,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:24,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:24,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:25,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:26,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:27,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:28,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:28,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:28,623 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:29,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:29,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:30,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:31,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:32,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:33,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:33,423 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:34,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:35,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:35,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:36,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:37,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:38,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:39,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:39,426 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:40,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:40,699 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:41,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:42,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:43,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:43,237 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:37:43,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:43,623 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:44,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:45,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:45,990 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:46,237 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:46,806 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:37:46,806 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 with policy now +2025-01-15 21:37:46,809 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:37:46,810 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:46,810 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:46,810 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:47,028 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/91eblnjt-media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:47,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:37:47,692 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/xz27t7gr-media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:48,261 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:48,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:49,254 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:49,431 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:50,282 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:51,261 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:51,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:52,272 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:53,259 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:53,432 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:54,256 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:55,287 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:55,433 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:56,292 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:57,009 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:57,306 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:58,313 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:58,630 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:58,630 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:58,631 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:58,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:58,661 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:59,329 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:59,435 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:59,435 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:00,312 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:01,334 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:01,436 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:02,359 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:02,877 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:03,313 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:04,332 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:05,346 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:05,437 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:06,378 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:07,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:08,376 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:08,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:09,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:10,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:11,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:11,440 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:12,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:13,238 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:38:13,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:13,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:13,627 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:13,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:14,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:15,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:16,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:17,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:18,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:19,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:19,717 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:20,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:21,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:22,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:23,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:24,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:24,718 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:25,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:26,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:27,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:28,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:28,627 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:28,627 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:29,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:29,762 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:30,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:31,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:32,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:33,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:34,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:35,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:35,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:36,357 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:37,358 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:37,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:37,505 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:38:37,505 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:38:37,506 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 with policy now +2025-01-15 21:38:37,506 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:38:37,506 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:38:37,506 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:38:37,770 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ezlyee49-media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:38,392 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:38,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:38,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:38,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:38:38,733 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/83pkxwjo-media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:39,409 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:39,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:40,380 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:40,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:40,737 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:41,380 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:42,373 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:43,238 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:38:43,379 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:43,453 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:43,659 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:43,659 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:44,383 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:45,408 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:46,421 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:46,433 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:47,417 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:47,455 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:48,440 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:49,446 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:50,421 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:51,437 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:51,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:52,446 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:53,434 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:54,440 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:54,458 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:38:54,701 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:38:54,701 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:38:54,701 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:38:55,436 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:55,458 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:38:55,459 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:56,480 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:57,447 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:57,459 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:57,573 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:58,470 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:58,657 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:58,658 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:59,454 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:00,465 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:01,449 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:01,461 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:02,493 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:02,750 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:03,464 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:04,460 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:05,460 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:05,463 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:06,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:07,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:07,465 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:08,136 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:08,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:09,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:10,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:11,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:12,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:13,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:39:13,240 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:13,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:13,630 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:13,631 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:14,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:15,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:16,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:17,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:17,470 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:18,325 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:18,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:19,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:20,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:21,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:22,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:23,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:23,748 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:24,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:25,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:26,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:27,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:28,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:28,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:28,631 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:28,793 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:29,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:30,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:31,478 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:32,351 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:39:32,351 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 with policy now +2025-01-15 21:39:32,357 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:39:32,358 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:39:32,358 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:39:32,367 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:39:32,477 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:32,576 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/q6owqepi-media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:32,836 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/djti9mvf-media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:33,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:33,510 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:34,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:34,523 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:34,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:35,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:35,493 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:36,497 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:37,478 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:37,521 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:38,496 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:39,491 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:40,353 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:40,509 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:41,480 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:41,520 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:42,531 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:43,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:39:43,527 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:43,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:43,651 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:44,530 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:45,534 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:45,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:45,759 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:39:45,759 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:39:45,759 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:39:46,482 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:39:46,482 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:39:46,524 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:46,759 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ks6w7yv7-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:39:47,483 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:47,530 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:48,558 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:49,483 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:49,577 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:50,550 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:51,116 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:51,484 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:51,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:52,561 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:53,564 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:54,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:55,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:56,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:56,806 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:57,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:58,614 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:58,652 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:58,653 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:59,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:00,581 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:01,489 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:01,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:01,960 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:02,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:03,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:04,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:05,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:06,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:07,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:07,781 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:08,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:09,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:10,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:11,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:12,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:12,781 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:13,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:40:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:13,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:13,638 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:14,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:15,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:16,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:17,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:17,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:18,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:19,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:20,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:21,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:22,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:22,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:24,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:25,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:26,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:27,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:28,020 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:28,365 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:40:28,365 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 with policy now +2025-01-15 21:40:28,379 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:40:28,380 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:40:28,380 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:40:28,380 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:40:28,593 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/poyozqqh-media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:28,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:28,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:28,688 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:28,858 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/o8xocbsf-media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:29,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:29,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:30,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:30,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:31,502 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:31,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:32,646 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:33,141 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:33,503 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:33,641 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:34,641 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:35,632 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:36,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:37,505 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:37,673 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:38,385 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:38,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:39,680 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:40,646 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:41,677 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:42,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:43,240 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:40:43,415 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:43,507 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:40:43,670 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:43,671 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:43,683 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:43,832 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:40:43,832 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:40:43,833 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:40:44,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:40:44,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:40:44,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:45,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:45,705 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:46,711 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:47,509 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:47,726 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:48,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:48,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:49,510 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:49,700 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:50,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:51,511 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:51,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:52,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:53,736 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:53,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:54,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:55,737 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:56,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:57,513 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:57,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:58,643 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:58,643 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:58,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:59,514 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:59,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:59,801 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:00,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:01,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:02,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:03,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:04,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:05,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:05,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:06,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:07,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:08,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:09,519 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:09,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:10,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:10,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:11,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:12,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:13,240 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:41:13,521 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:13,643 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:13,643 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:13,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:14,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:15,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:16,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:16,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:17,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:18,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:19,715 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:20,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:21,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:21,803 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:22,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:23,734 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:24,746 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:24,948 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:41:24,948 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 with policy now +2025-01-15 21:41:24,952 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:41:24,952 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:41:24,952 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:41:24,953 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:41:25,181 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fyspcu3h-media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:41:25,747 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:25,797 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ev5j11ae-media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:26,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:26,754 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:27,063 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:27,527 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:27,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:28,703 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:28,703 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:28,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:29,783 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:30,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:31,528 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:31,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:32,070 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:32,800 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:33,529 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:33,779 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:34,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:35,530 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:41:35,804 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:35,898 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:41:35,898 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:41:35,898 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:41:36,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:41:36,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:41:36,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:37,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:37,745 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:37,817 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:38,816 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:39,532 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:39,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:40,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:41,533 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:41,809 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:42,842 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:43,089 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:43,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:41:43,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:43,693 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:43,859 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:44,835 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:45,855 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:46,855 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:47,873 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:48,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:48,836 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:49,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:50,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:51,538 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:51,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:52,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:53,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:53,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:54,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:55,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:56,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:57,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:58,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:58,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:58,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:58,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:59,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:00,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:01,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:02,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:03,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:04,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:04,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:05,857 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:06,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:07,277 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:07,277 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 with policy now +2025-01-15 21:42:07,296 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:07,296 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:07,296 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:07,297 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:07,489 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/5qnttwb1-media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:42:07,806 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/t7o4qhg3-media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,856 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:08,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:08,886 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:09,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:09,850 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:09,858 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:10,875 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:11,859 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:12,873 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:13,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:13,548 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:13,676 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:13,676 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:13,890 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:14,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:15,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:15,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:16,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:17,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:17,909 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:17,947 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:17,948 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:17,948 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:18,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:18,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:18,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:19,174 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 21:42:19,174 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 21:42:19,327 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval +2025-01-15 21:42:19,638 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppigqiw9w +2025-01-15 21:42:19,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 21:42:19,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:19,767 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fz6rnj1y-media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:19,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:20,085 INFO SenderThread:2467459 [sender.py:send_request_log_artifact():1525] logged artifact run-vfeb54wi-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MTY2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTEzNDQyMA==', 'latestArtifact': None}} +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:20,085 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json with policy now +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,086 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,086 DEBUG SenderThread:2467459 [sender.py:send():379] send: telemetry +2025-01-15 21:42:20,086 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,090 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,090 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,091 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,091 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,091 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,091 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,092 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,092 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,092 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,092 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,093 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,093 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,093 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,094 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,094 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,094 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,095 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,095 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,095 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,095 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,096 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,096 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 21:42:20,334 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ah1sq9kq-media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:20,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval +2025-01-15 21:42:20,595 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpknxrse51 +2025-01-15 21:42:20,741 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/vxvgjgrh-media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:20,799 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ttd8s49d-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:20,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:21,036 INFO SenderThread:2467459 [sender.py:send_request_log_artifact():1525] logged artifact run-vfeb54wi-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MTY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTEzNDQyMQ==', 'latestArtifact': None}} +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:21,037 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:21,037 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json with policy now +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:21,038 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:21,280 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/v1krkjej-media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:21,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:21,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:21,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:22,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:23,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:24,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:25,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:26,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:26,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:27,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:28,649 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:28,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:28,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:29,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:30,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:31,823 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:31,918 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:32,557 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:42:32,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:33,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:34,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:35,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:36,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:37,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:37,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:38,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:39,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:40,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:41,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:42,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:42,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:43,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:43,649 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:43,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:43,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:44,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:45,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:46,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:47,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:47,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:48,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:49,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:50,846 DEBUG SenderThread:2467459 [sender.py:send():379] send: exit +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():586] handling exit code: 0 +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():588] handling runtime: 1057 +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():594] send defer +2025-01-15 21:42:50,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,846 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-15 21:42:50,847 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 1 +2025-01-15 21:42:50,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-15 21:42:50,847 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 2 +2025-01-15 21:42:50,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [system_monitor.py:finish():203] Stopping system monitor +2025-01-15 21:42:50,847 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-15 21:42:50,847 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-15 21:42:50,849 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined cpu monitor +2025-01-15 21:42:50,849 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined disk monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined gpu monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined memory monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined network monitor +2025-01-15 21:42:50,914 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,914 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-15 21:42:50,914 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 3 +2025-01-15 21:42:50,914 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,914 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:50,914 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-15 21:42:50,915 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,915 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-15 21:42:50,915 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 4 +2025-01-15 21:42:50,916 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,916 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-15 21:42:51,567 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:53,568 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: keepalive +2025-01-15 21:42:56,959 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,959 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-15 21:42:56,959 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 5 +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:56,960 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,960 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-15 21:42:56,961 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:56,961 DEBUG SenderThread:2467459 [sender.py:send():379] send: summary +2025-01-15 21:42:56,961 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 6 +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,962 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 7 +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,962 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-15 21:42:57,570 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:57,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:42:59,953 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 8 +2025-01-15 21:42:59,953 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:42:59,954 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:59,954 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-15 21:42:59,954 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:59,954 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-15 21:42:59,954 INFO SenderThread:2467459 [job_builder.py:build():440] Attempting to build job artifact +2025-01-15 21:42:59,957 INFO SenderThread:2467459 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-15 21:42:59,959 INFO SenderThread:2467459 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-15 21:42:59,976 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 9 +2025-01-15 21:42:59,976 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:59,976 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-15 21:42:59,976 DEBUG SenderThread:2467459 [sender.py:send():379] send: artifact +2025-01-15 21:43:00,572 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:43:00,626 INFO wandb-upload_0:2467459 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphw5sjkl5 +2025-01-15 21:43:00,848 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:00,891 INFO wandb-upload_2:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk_xp6b81 +2025-01-15 21:43:00,941 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /tmp/tmpx6bsw9uf/wandb-job.json +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-15 21:43:01,577 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml config.yaml +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json wandb-summary.json +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt requirements.txt +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json wandb-metadata.json +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log output.log +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch diff.patch +2025-01-15 21:43:02,575 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:43:02,577 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 10 +2025-01-15 21:43:02,582 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:02,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,582 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-15 21:43:02,582 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,582 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-15 21:43:02,582 INFO SenderThread:2467459 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 21:43:02,756 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:43:02,759 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt +2025-01-15 21:43:02,760 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:43:02,789 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:43:02,849 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:02,849 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:02,989 INFO Thread-11 (_thread_body):2467459 [sender.py:transition_state():614] send defer: 11 +2025-01-15 21:43:02,989 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,989 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-15 21:43:02,989 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,990 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-15 21:43:02,990 INFO SenderThread:2467459 [file_pusher.py:join():175] waiting for file pusher +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 12 +2025-01-15 21:43:02,993 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,993 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-15 21:43:02,993 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [file_stream.py:finish():601] file stream finish called +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [file_stream.py:finish():605] file stream finish is done +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 13 +2025-01-15 21:43:03,229 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:03,229 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-15 21:43:03,229 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 14 +2025-01-15 21:43:03,229 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:03,230 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: final +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: footer +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:03,230 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-15 21:43:03,230 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:03,231 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: server_info +2025-01-15 21:43:03,231 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:03,231 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: server_info +2025-01-15 21:43:03,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:03,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: get_summary +2025-01-15 21:43:03,236 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-15 21:43:03,237 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:43:03,295 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:03,296 INFO MainThread:2467459 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-15 21:43:03,296 INFO MainThread:2467459 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-15 21:43:03,297 INFO MainThread:2467459 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-15 21:43:03,297 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: shutdown +2025-01-15 21:43:03,297 INFO HandlerThread:2467459 [handler.py:finish():882] shutting down handler +2025-01-15 21:43:04,235 INFO WriterThread:2467459 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [sender.py:finish():1615] shutting down sender +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [file_pusher.py:join():175] waiting for file pusher diff --git a/vfeb54wi/wandb/debug.log b/vfeb54wi/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c1b8673697e695a086b02d4bda70ce40bafd4a37 --- /dev/null +++ b/vfeb54wi/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Configure stats pid to 2467097 +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/settings +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug.log +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug-internal.log +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():569] calling init triggers +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi'} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():619] starting backend +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():623] setting up manager +2025-01-15 21:25:12,917 INFO MainThread:2467097 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-15 21:25:12,918 INFO MainThread:2467097 [wandb_init.py:init():631] backend started and connected +2025-01-15 21:25:12,928 INFO MainThread:2467097 [wandb_init.py:init():720] updated telemetry +2025-01-15 21:25:12,936 INFO MainThread:2467097 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-15 21:25:13,164 INFO MainThread:2467097 [wandb_run.py:_on_init():2435] communicating current version +2025-01-15 21:25:13,213 INFO MainThread:2467097 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-15 21:25:13,213 INFO MainThread:2467097 [wandb_init.py:init():804] starting run threads in backend +2025-01-15 21:25:13,583 INFO MainThread:2467097 [wandb_run.py:_console_start():2413] atexit reg +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-15 21:25:13,586 INFO MainThread:2467097 [wandb_init.py:init():847] run started, returning control to user process +2025-01-15 21:25:13,586 INFO MainThread:2467097 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/tb, True +2025-01-15 21:25:18,430 INFO MainThread:2467097 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-15 21:43:04,298 WARNING MsgRouterThr:2467097 [router.py:message_loop():77] message_loop has been closed diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efe7689195040772376e371d6a909ec5bb67b461 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_0' +wandb_notes: + desc: null + value: 3456_0_req_3921_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736994312 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..6a47c0ed68a9c65b71fd74d87bee4cd94105b0dd --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch @@ -0,0 +1,74 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..2659f59 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,7 +133,7 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-wandb_run_id: '1002_0' ++wandb_run_id: '3456_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..134df65 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0324917e3fe8998249500c7ed7a5b5ba3f8f2eb8 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.75, 0.47330674529609484, 0.9187985359784809, 0.5133531242582317, 0.889095514934922, 0.5846942215988524, 0.8802883326246717, 0.6031669080051083, 0.8712705745550526]]} \ No newline at end of file diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3efcafb15cdd1545186b2daa7711e9b506af9605 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[15, 20, 0.75, 0.4983611702922284, 0.9230575858700546, 0.5425259619866196, 0.8882155864654401, 0.5970279723211784, 0.8681355416038479, 0.585568152350955, 0.862504541801431]]} \ No newline at end of file diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9139fa86d84dd3000936f3d449001f2ff114f7c --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1b728d66af3d385c7669479799a18979a7f2ca22d2f8b5b40e16ebc8b985e7 +size 1154574 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..925f75abd4f42a4ea42f31c94c5837ea20eff8b6 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be070cda4484f5d8f382748fd4c2f4120827e145f3b8d4e97bd4c7a1afb47ab1 +size 1184944 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9b0959d9dbbd2e500b1e0a3d63e899bb26a2b8ec --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f892c1875d8f769e839d92205b3decdca4a636b2e461ce5fc10bcb7228b6a03 +size 1343488 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a92778174d96e5ad8d41d9cc584fbeab00538049 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7f16c34c55c022ec9cf4d75084ba2f749e4b014ef3b58dd31fb9421f10d545c +size 1176474 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2de15a8e5883ff4a495558671f59f0cc5048e6b7 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b286629d17bfd49bd46bf22225f31e406eb5d7e263ee57e66fb39384533cd8c3 +size 1033226 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62a54d119f033f8032e3eac8395be02a0c52f905 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:389de3f12b22a271b1c9aadc06c937ff73766ce06d34e3f387c013c7057e77f2 +size 1185026 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..05e6ee516977b576776ce4fce11fe045f52b2fd9 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57fbf8f750a05cbd03dd04598b04fce82ad4cc8b09331e81fc130bd77afd9af2 +size 1393370 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e9205b5a8dbc27d1e7b303859e9cd66f838675b --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c066e7ff5f521f1bd1c39a3036a4b73720e839fd83caae03a75e0722a74bf27f +size 1359276 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8d318ed7d544a328a4a65dff5f006c0e3d82300 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e0251ea6b3f5a1ef81f55f14293484d05071c1988ddd946dfd98d6feb148a3a +size 1517963 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bec0da4542e95e6ff42a7c5c43217ede189f66d --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792d83e6a625ca80ced859abef65635aa41d06fab9a802b2f960400386a3751f +size 1464090 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..41916924492b3fef800d9cd2176bee811bdd2011 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c347402d83b94d0d76b5e3a7d52f138909cd08d1025f74ebc5b9ca9b69ed8972 +size 1188193 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f3258389251f7a632edd397774e4bc558869a6f4 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b58a2f9660a5285887a3d20341f0c589fb20b2547551eaeeae423e564854e87 +size 1157372 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3bfc1fb84f0c24821201347371377cfb3d81daf3 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb336114555212728d0bcb3962112f5381481464e1262f66172e00265000a3af +size 1151051 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a320b2a4ed652d45ed8e8ff636afe5502ecda81 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327a1cef6154eb393524a9a4d9524bf91fb1a9c609b08188bf2291d815804422 +size 1163054 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68d341c5d0a5713b7f6fbeafcc830f3f983f0f6a --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65dbd3f5364b11e95697bbbe843f808302fe6de1dd8970e11d34656b81a9ce5f +size 1162281 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17c98412e4efe0cc247bc6a91bc3670dc244ff19 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7562c03fef4135eb5f5617db1ec1d73d6999e35e6076fb7fa99289eafa858b90 +size 753000 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0707e43a3487c76e4ef31fae94303624ea151c1d --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54affceed9218be400c4da12e96bab1b3c4df28d7fe42c121166fe3b49de3b15 +size 707515 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ae786436edf25d88fb65a30929d4dd55820dae5 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8602e79d56991114b037d8b0ab35aa65bc3e1df29aae727eac558f989ea4e07d +size 694129 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c6bb6e07d6c910dbcb3a71db3655eeb043d2c3d3 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ccbfe16dcecf0f3420f42716100ebe30013ff97399f08262016d1fb3d5f124a +size 1174029 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c3b890fcdf4e3a875451172538a23dd202189eea --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f85b9e56cea35531fc0f61edc8b99dd9cdb152c2de405de2efc7cab609d4181 +size 1176285 diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e59c8ef1ce7db672743057d08e371043d6bd4b67 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log @@ -0,0 +1,1123 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736994318.431723]: resetting environment +[INFO] [1736994318.437702]: cleared current plan +[INFO] [1736994318.439249]: moving to home +[INFO] [1736994321.243010]: reached home +[INFO] [1736994321.244246]: reset action history +[INFO] [1736994323.952796]: environment reset +Starting episode 0 +[INFO] [1736994323.955074]: resetting environment +[INFO] [1736994323.963314]: cleared current plan +[INFO] [1736994323.964273]: moving to home +[INFO] [1736994324.967194]: reached home +[INFO] [1736994324.968801]: reset action history +[INFO] [1736994327.677241]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736994387.325639]: resetting environment +[INFO] [1736994387.329109]: cleared current plan +[INFO] [1736994387.329951]: moving to home +[INFO] [1736994388.332605]: reached home +[INFO] [1736994388.333813]: reset action history +[INFO] [1736994391.041835]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_0_2025-01-15-21-26-31.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:38429/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2467613] +process[print_text-2]: started with pid [2467637] +process[rosbag_record-3]: started with pid [2467638] +started bagging! +For topic gripper_width: timestamp difference is 101052726 for nearest: 1736994394187267268 - target: 1736994394086214542 at idx 47 +[INFO] [1736994418.049850]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1736994418.763572]: Stored episode 1. +[INFO] [1736994418.763773]: Saving video... +[INFO] [1736994418.963007]: Video saved! +Starting episode 1 +[INFO] [1736994418.974371]: resetting environment +[INFO] [1736994418.978648]: cleared current plan +[INFO] [1736994418.979867]: moving to home +[INFO] [1736994422.582672]: reached home +[INFO] [1736994422.583129]: reset action history +[INFO] [1736994425.290898]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994449.129460]: resetting environment +[INFO] [1736994449.134274]: cleared current plan +[INFO] [1736994449.134652]: moving to home +[INFO] [1736994450.135978]: reached home +[INFO] [1736994450.136397]: reset action history +[INFO] [1736994452.893822]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_1_2025-01-15-21-27-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:46833/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2467893] +process[print_text-5]: started with pid [2467894] +process[rosbag_record-6]: started with pid [2467918] +started bagging! +For topic gripper_width: timestamp difference is -205503622 for nearest: 1736994467387217125 - target: 1736994467592720747 at idx 204 +For topic gripper_width: timestamp difference is 100280064 for nearest: 1736994474687259453 - target: 1736994474586979389 at idx 249 +For topic gripper_width: timestamp difference is 102194939 for nearest: 1736994477087260399 - target: 1736994476985065460 at idx 249 +For topic gripper_width: timestamp difference is 102809670 for nearest: 1736994477887210848 - target: 1736994477784401178 at idx 249 +For topic gripper_width: timestamp difference is 103218197 for nearest: 1736994478287290215 - target: 1736994478184072018 at idx 249 +[INFO] [1736994479.884738]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736994480.634656]: Stored episode 2. +[INFO] [1736994480.636534]: Saving video... +[INFO] [1736994481.042897]: Video saved! +Starting episode 2 +[INFO] [1736994481.087816]: resetting environment +[INFO] [1736994481.098393]: cleared current plan +[INFO] [1736994481.098623]: moving to home +[INFO] [1736994484.701617]: reached home +[INFO] [1736994484.705129]: reset action history +[INFO] [1736994487.481911]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994503.126759]: resetting environment +[INFO] [1736994503.133788]: cleared current plan +[INFO] [1736994503.134904]: moving to home +[INFO] [1736994504.136953]: reached home +[INFO] [1736994504.137453]: reset action history +[INFO] [1736994506.845457]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_2_2025-01-15-21-28-26.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:41997/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2468038] +process[print_text-8]: started with pid [2468039] +process[rosbag_record-9]: started with pid [2468063] +started bagging! +For topic gripper_width: timestamp difference is 101448240 for nearest: 1736994517453880729 - target: 1736994517352432489 at idx 146 +[INFO] [1736994533.817967]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736994534.603339]: Stored episode 3. +[INFO] [1736994534.608793]: Saving video... +[INFO] [1736994535.220609]: Video saved! +Starting episode 3 +[INFO] [1736994535.284549]: resetting environment +[INFO] [1736994535.295281]: cleared current plan +[INFO] [1736994535.296865]: moving to home +[WARN] [1736994535.166824]: Plan exhausted +[WARN] [1736994535.206918]: Plan exhausted +[WARN] [1736994535.248122]: Plan exhausted +[WARN] [1736994535.286792]: Plan exhausted +[INFO] [1736994538.916630]: reached home +[INFO] [1736994538.923056]: reset action history +[INFO] [1736994541.648978]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994548.705826]: resetting environment +[INFO] [1736994548.711357]: cleared current plan +[INFO] [1736994548.711663]: moving to home +[INFO] [1736994549.716275]: reached home +[INFO] [1736994549.720578]: reset action history +[INFO] [1736994552.497912]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_3_2025-01-15-21-29-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:41787/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2468173] +process[print_text-11]: started with pid [2468174] +process[rosbag_record-12]: started with pid [2468198] +started bagging! +[INFO] [1736994579.455587]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736994580.227918]: Stored episode 4. +[INFO] [1736994580.230891]: Saving video... +[INFO] [1736994580.921986]: Video saved! +Starting episode 4 +[INFO] [1736994580.978452]: resetting environment +[INFO] [1736994580.997303]: cleared current plan +[INFO] [1736994580.997907]: moving to home +[WARN] [1736994580.813514]: Plan exhausted +[WARN] [1736994580.846966]: Plan exhausted +[WARN] [1736994580.886851]: Plan exhausted +[WARN] [1736994580.927330]: Plan exhausted +[WARN] [1736994580.968640]: Plan exhausted +[INFO] [1736994584.602957]: reached home +[INFO] [1736994584.606329]: reset action history +[INFO] [1736994587.405489]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994594.356481]: resetting environment +[INFO] [1736994594.378453]: cleared current plan +[INFO] [1736994594.383554]: moving to home +[INFO] [1736994595.389426]: reached home +[INFO] [1736994595.391239]: reset action history +[INFO] [1736994598.114241]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_4_2025-01-15-21-29-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:36049/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2468316] +process[print_text-14]: started with pid [2468317] +process[rosbag_record-15]: started with pid [2468341] +started bagging! +[INFO] [1736994625.081279]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736994625.870348]: Stored episode 5. +[INFO] [1736994625.881377]: Saving video... +[INFO] [1736994626.199456]: Video saved! +Starting episode 5 +[INFO] [1736994626.251486]: resetting environment +[INFO] [1736994626.267472]: cleared current plan +[INFO] [1736994626.268235]: moving to home +[INFO] [1736994629.877204]: reached home +[INFO] [1736994629.878549]: reset action history +[INFO] [1736994632.729970]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994639.780787]: resetting environment +[INFO] [1736994639.808634]: cleared current plan +[INFO] [1736994639.815354]: moving to home +[INFO] [1736994640.820470]: reached home +[INFO] [1736994640.825277]: reset action history +[INFO] [1736994643.563439]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_5_2025-01-15-21-30-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:41747/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2468485] +process[print_text-17]: started with pid [2468486] +process[rosbag_record-18]: started with pid [2468487] +started bagging! +For topic gripper_width: timestamp difference is -155495481 for nearest: 1736994652787259264 - target: 1736994652942754745 at idx 120 +preempting episode +[INFO] [1736994659.713467]: Storing episode... +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[pub_text-16] killing on exit +[INFO] [1736994660.549624]: Stored episode 6. +[INFO] [1736994660.555809]: Saving video... +[INFO] [1736994661.030964]: Video saved! +Starting episode 6 +[INFO] [1736994661.085293]: resetting environment +[INFO] [1736994661.112694]: cleared current plan +[INFO] [1736994661.115064]: moving to home +[WARN] [1736994660.966821]: Plan exhausted +[WARN] [1736994661.009062]: Plan exhausted +[WARN] [1736994661.047629]: Plan exhausted +[WARN] [1736994661.090056]: Plan exhausted +[INFO] [1736994664.722108]: reached home +[INFO] [1736994664.726995]: reset action history +[INFO] [1736994667.560466]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994676.847984]: resetting environment +[INFO] [1736994676.880331]: cleared current plan +[INFO] [1736994676.889337]: moving to home +[INFO] [1736994677.897656]: reached home +[INFO] [1736994677.902886]: reset action history +[INFO] [1736994680.743213]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_6_2025-01-15-21-31-20.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:42171/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2468627] +process[print_text-20]: started with pid [2468628] +process[rosbag_record-21]: started with pid [2468629] +started bagging! +preempting episode +[INFO] [1736994695.740847]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[pub_text-19] killing on exit +[INFO] [1736994696.683553]: Stored episode 7. +[INFO] [1736994696.684951]: Saving video... +[INFO] [1736994697.345907]: Video saved! +Starting episode 7 +[INFO] [1736994697.427220]: resetting environment +[INFO] [1736994697.470383]: cleared current plan +[INFO] [1736994697.480126]: moving to home +[WARN] [1736994697.051672]: Plan exhausted +[WARN] [1736994697.087574]: Plan exhausted +[WARN] [1736994697.126970]: Plan exhausted +[WARN] [1736994697.167522]: Plan exhausted +[WARN] [1736994697.210389]: Plan exhausted +[WARN] [1736994697.246975]: Plan exhausted +[WARN] [1736994697.286836]: Plan exhausted +[WARN] [1736994697.327175]: Plan exhausted +[WARN] [1736994697.367629]: Plan exhausted +[WARN] [1736994697.406805]: Plan exhausted +[WARN] [1736994697.447092]: Plan exhausted +[INFO] [1736994700.995351]: reached home +[INFO] [1736994701.003880]: reset action history +[INFO] [1736994703.871427]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994710.355152]: resetting environment +[INFO] [1736994710.383571]: cleared current plan +[INFO] [1736994710.387296]: moving to home +[INFO] [1736994711.396426]: reached home +[INFO] [1736994711.412738]: reset action history +[INFO] [1736994714.273241]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_7_2025-01-15-21-31-54.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:36953/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2468759] +process[print_text-23]: started with pid [2468760] +process[rosbag_record-24]: started with pid [2468761] +started bagging! +preempting episode +[INFO] [1736994728.928333]: Storing episode... +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[WARN] [1736994730.249684]: Plan exhausted +[WARN] [1736994730.286856]: Plan exhausted +[INFO] [1736994729.749320]: Stored episode 8. +[INFO] [1736994729.762683]: Saving video... +[INFO] [1736994730.185459]: Video saved! +Starting episode 8 +[INFO] [1736994730.273945]: resetting environment +[INFO] [1736994730.300183]: cleared current plan +[INFO] [1736994730.305037]: moving to home +[INFO] [1736994733.835380]: reached home +[INFO] [1736994733.858677]: reset action history +[INFO] [1736994736.787421]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994756.156981]: resetting environment +[INFO] [1736994756.188700]: cleared current plan +[INFO] [1736994756.195994]: moving to home +[INFO] [1736994757.212535]: reached home +[INFO] [1736994757.232511]: reset action history +[INFO] [1736994760.168098]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_8_2025-01-15-21-32-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:44909/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2468906] +process[print_text-26]: started with pid [2468907] +process[rosbag_record-27]: started with pid [2468908] +started bagging! +For topic gripper_width: timestamp difference is -100260863 for nearest: 1736994763387284627 - target: 1736994763487545490 at idx 59 +For topic gripper_width: timestamp difference is 100129843 for nearest: 1736994763787283659 - target: 1736994763687153816 at idx 61 +[INFO] [1736994787.370249]: Storing episode... +[print_text-26] killing on exit +[rosbag_record-27] killing on exit +[pub_text-25] killing on exit +[WARN] [1736994788.745363]: Plan exhausted +[WARN] [1736994788.786089]: Plan exhausted +[INFO] [1736994788.181412]: Stored episode 9. +[INFO] [1736994788.192954]: Saving video... +[INFO] [1736994788.655235]: Video saved! +Starting episode 9 +[INFO] [1736994788.767914]: resetting environment +[INFO] [1736994788.798812]: cleared current plan +[INFO] [1736994788.802443]: moving to home +[INFO] [1736994792.412235]: reached home +[INFO] [1736994792.418925]: reset action history +[INFO] [1736994795.385843]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994801.420125]: resetting environment +[INFO] [1736994801.478557]: cleared current plan +[INFO] [1736994801.483979]: moving to home +[INFO] [1736994802.505536]: reached home +[INFO] [1736994802.507433]: reset action history +[INFO] [1736994805.432613]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_9_2025-01-15-21-33-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:37873/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2469047] +process[print_text-29]: started with pid [2469048] +process[rosbag_record-30]: started with pid [2469049] +started bagging! +For topic gripper_width: timestamp difference is 102569784 for nearest: 1736994806853926863 - target: 1736994806751357079 at idx 27 +For topic gripper_width: timestamp difference is 101388483 for nearest: 1736994814053925781 - target: 1736994813952537298 at idx 127 +[INFO] [1736994832.636363]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit[pub_text-28] killing on exit +[INFO] [1736994833.440842]: Stored episode 10. +[INFO] [1736994833.469959]: Saving video... +[WARN] [1736994833.971205]: Plan exhausted +[WARN] [1736994834.011508]: Plan exhausted +[WARN] [1736994834.047983]: Plan exhausted +[WARN] [1736994834.090740]: Plan exhausted +[WARN] [1736994834.127414]: Plan exhausted +[WARN] [1736994834.171469]: Plan exhausted +[WARN] [1736994834.213743]: Plan exhausted +[WARN] [1736994834.247267]: Plan exhausted +[WARN] [1736994834.288438]: Plan exhausted +[WARN] [1736994834.330621]: Plan exhausted +[INFO] [1736994834.199497]: Video saved! +Starting episode 10 +[INFO] [1736994834.342944]: resetting environment +[INFO] [1736994834.364727]: cleared current plan +[INFO] [1736994834.371732]: moving to home +[INFO] [1736994837.994656]: reached home +[INFO] [1736994838.012255]: reset action history +[INFO] [1736994840.981798]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994845.772252]: resetting environment +[INFO] [1736994845.817440]: cleared current plan +[INFO] [1736994845.821055]: moving to home +[INFO] [1736994846.844025]: reached home +[INFO] [1736994846.846256]: reset action history +[INFO] [1736994849.845669]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_10_2025-01-15-21-34-09.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:41721/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2469182] +process[print_text-32]: started with pid [2469183] +process[rosbag_record-33]: started with pid [2469184] +started bagging! +[INFO] [1736994877.070357]: Storing episode... +[print_text-32] killing on exit +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[WARN] [1736994878.207036]: Plan exhausted +[WARN] [1736994878.246802]: Plan exhausted +[WARN] [1736994878.289437]: Plan exhausted +[WARN] [1736994878.326804]: Plan exhausted +[WARN] [1736994878.367386]: Plan exhausted +[WARN] [1736994878.409417]: Plan exhausted +[WARN] [1736994878.451744]: Plan exhausted +[INFO] [1736994877.883550]: Stored episode 11. +[INFO] [1736994877.885436]: Saving video... +[INFO] [1736994878.308449]: Video saved! +Starting episode 11 +[INFO] [1736994878.443433]: resetting environment +[INFO] [1736994878.485727]: cleared current plan +[INFO] [1736994878.486815]: moving to home +[INFO] [1736994882.098344]: reached home +[INFO] [1736994882.110917]: reset action history +[INFO] [1736994885.247715]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994889.796741]: resetting environment +[INFO] [1736994889.841067]: cleared current plan +[INFO] [1736994889.866412]: moving to home +[INFO] [1736994890.884715]: reached home +[INFO] [1736994890.924917]: reset action history +[INFO] [1736994893.997719]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_11_2025-01-15-21-34-54.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:34597/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2469326] +process[print_text-35]: started with pid [2469327] +process[rosbag_record-36]: started with pid [2469328] +started bagging! +[INFO] [1736994925.791168]: Storing episode... +[print_text-35] killing on exit +[pub_text-34] killing on exit +[rosbag_record-36] killing on exit +[INFO] [1736994926.667544]: Stored episode 12. +[INFO] [1736994926.673883]: Saving video... +[INFO] [1736994927.092091]: Video saved! +Starting episode 12 +[INFO] [1736994927.239930]: resetting environment +[INFO] [1736994927.278641]: cleared current plan +[INFO] [1736994927.285393]: moving to home +[WARN] [1736994927.133349]: Plan exhausted +[WARN] [1736994927.182674]: Plan exhausted +[WARN] [1736994927.212554]: Plan exhausted +[WARN] [1736994927.253478]: Plan exhausted +[INFO] [1736994931.005272]: reached home +[INFO] [1736994931.017714]: reset action history +[INFO] [1736994934.141650]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994943.473247]: resetting environment +[INFO] [1736994943.576375]: cleared current plan +[INFO] [1736994943.594234]: moving to home +[INFO] [1736994944.601719]: reached home +[INFO] [1736994944.618591]: reset action history +[INFO] [1736994947.705976]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_12_2025-01-15-21-35-47.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:34977/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2469460] +process[print_text-38]: started with pid [2469461] +process[rosbag_record-39]: started with pid [2469462] +started bagging! +For topic gripper_width: timestamp difference is 103123551 for nearest: 1736994969153882390 - target: 1736994969050758839 at idx 249 +For topic gripper_width: timestamp difference is 103798249 for nearest: 1736994972753922799 - target: 1736994972650124550 at idx 249 +[INFO] [1736994974.948004]: Storing episode... +[pub_text-37] killing on exit +[rosbag_record-39] killing on exit +[print_text-38] killing on exit +[INFO] [1736994975.785120]: Stored episode 13. +[INFO] [1736994975.802905]: Saving video... +[WARN] [1736994976.298783]: Plan exhausted +[WARN] [1736994976.336881]: Plan exhausted +[WARN] [1736994976.367818]: Plan exhausted +[WARN] [1736994976.410133]: Plan exhausted +[WARN] [1736994976.448191]: Plan exhausted +[WARN] [1736994976.489565]: Plan exhausted +[WARN] [1736994976.528190]: Plan exhausted +[WARN] [1736994976.569477]: Plan exhausted +[WARN] [1736994976.607339]: Plan exhausted +[WARN] [1736994976.646816]: Plan exhausted +[INFO] [1736994976.502771]: Video saved! +Starting episode 13 +[INFO] [1736994976.610075]: resetting environment +[INFO] [1736994976.668973]: cleared current plan +[INFO] [1736994976.672073]: moving to home +[INFO] [1736994980.383148]: reached home +[INFO] [1736994980.395248]: reset action history +[INFO] [1736994983.498192]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736994991.775336]: resetting environment +[INFO] [1736994991.824238]: cleared current plan +[INFO] [1736994991.855127]: moving to home +[INFO] [1736994992.889432]: reached home +[INFO] [1736994992.911717]: reset action history +[INFO] [1736994995.891235]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_13_2025-01-15-21-36-35.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:37811/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2469603] +process[print_text-41]: started with pid [2469604] +process[rosbag_record-42]: started with pid [2469605] +started bagging! +preempting episode +[INFO] [1736995018.598909]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[INFO] [1736995019.431647]: Stored episode 14. +[INFO] [1736995019.450703]: Saving video... +[INFO] [1736995019.914304]: Video saved! +[WARN] [1736995019.863503]: Plan exhausted +[WARN] [1736995019.886870]: Plan exhausted +[WARN] [1736995019.931819]: Plan exhausted +[WARN] [1736995019.968193]: Plan exhausted +[WARN] [1736995020.006796]: Plan exhausted +[WARN] [1736995020.049581]: Plan exhausted +Starting episode 14 +[INFO] [1736995020.032695]: resetting environment +[INFO] [1736995020.078800]: cleared current plan +[INFO] [1736995020.087113]: moving to home +[INFO] [1736995023.714079]: reached home +[INFO] [1736995023.725810]: reset action history +[INFO] [1736995026.821965]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995033.554279]: resetting environment +[INFO] [1736995033.637503]: cleared current plan +[INFO] [1736995033.660055]: moving to home +[INFO] [1736995034.691654]: reached home +[INFO] [1736995034.707348]: reset action history +[INFO] [1736995037.890953]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_14_2025-01-15-21-37-17.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:44425/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2469734] +process[print_text-44]: started with pid [2469735] +process[rosbag_record-45]: started with pid [2469736] +started bagging! +For topic gripper_width: timestamp difference is 100340875 for nearest: 1736995050587245496 - target: 1736995050486904621 at idx 176 +For topic gripper_width: timestamp difference is 105458734 for nearest: 1736995057787281511 - target: 1736995057681822777 at idx 249 +[INFO] [1736995064.963460]: Storing episode... +[rosbag_record-45] killing on exit +[print_text-44] killing on exit +[pub_text-43] killing on exit +[INFO] [1736995065.972189]: Stored episode 15. +[INFO] [1736995066.017707]: Saving video... +[WARN] [1736995066.326824]: Plan exhausted +[WARN] [1736995066.373982]: Plan exhausted +[WARN] [1736995066.406819]: Plan exhausted +[WARN] [1736995066.455619]: Plan exhausted +[WARN] [1736995066.488964]: Plan exhausted +[WARN] [1736995066.533092]: Plan exhausted +[WARN] [1736995066.567098]: Plan exhausted +[WARN] [1736995066.612521]: Plan exhausted +[WARN] [1736995066.646808]: Plan exhausted +[WARN] [1736995066.690717]: Plan exhausted +[WARN] [1736995066.729189]: Plan exhausted +[WARN] [1736995066.783430]: Plan exhausted +[WARN] [1736995066.811286]: Plan exhausted +[WARN] [1736995066.860453]: Plan exhausted +[INFO] [1736995066.667136]: Video saved! +Starting episode 15 +[INFO] [1736995066.834296]: resetting environment +[INFO] [1736995066.879409]: cleared current plan +[INFO] [1736995066.879714]: moving to home +[INFO] [1736995070.585219]: reached home +[INFO] [1736995070.605935]: reset action history +[INFO] [1736995073.959257]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995078.701247]: resetting environment +[INFO] [1736995078.813796]: cleared current plan +[INFO] [1736995078.815226]: moving to home +[INFO] [1736995079.829204]: reached home +[INFO] [1736995079.851475]: reset action history +[INFO] [1736995083.056301]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_15_2025-01-15-21-38-03.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:35097/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2469881] +process[print_text-47]: started with pid [2469882] +process[rosbag_record-48]: started with pid [2469883] +started bagging! +[INFO] [1736995116.089024]: Storing episode... +[print_text-47] killing on exit +[rosbag_record-48] killing on exit +[pub_text-46] killing on exit +[INFO] [1736995116.935099]: Stored episode 16. +[INFO] [1736995116.957289]: Saving video... +[INFO] [1736995117.345222]: Video saved! +Starting episode 16 +[INFO] [1736995117.479086]: resetting environment +[INFO] [1736995117.524779]: cleared current plan +[INFO] [1736995117.535640]: moving to home +[WARN] [1736995117.448071]: Plan exhausted +[WARN] [1736995117.490153]: Plan exhausted +[INFO] [1736995121.252188]: reached home +[INFO] [1736995121.279600]: reset action history +[INFO] [1736995124.385281]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995134.489402]: resetting environment +[INFO] [1736995134.512638]: cleared current plan +[INFO] [1736995134.514819]: moving to home +[INFO] [1736995135.522543]: reached home +[INFO] [1736995135.534206]: reset action history +[INFO] [1736995138.659662]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_16_2025-01-15-21-38-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:38667/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2470024] +process[print_text-50]: started with pid [2470025] +process[rosbag_record-51]: started with pid [2470026] +started bagging! +For topic gripper_width: timestamp difference is -115692868 for nearest: 1736995155087258086 - target: 1736995155202950954 at idx 234 +[INFO] [1736995170.841197]: Storing episode... +[pub_text-49] killing on exit +[print_text-50] killing on exit +[rosbag_record-51] killing on exit +[INFO] [1736995171.713895]: Stored episode 17. +[INFO] [1736995171.718241]: Saving video... +[WARN] [1736995172.178294]: Plan exhausted +[WARN] [1736995172.213616]: Plan exhausted +[WARN] [1736995172.247688]: Plan exhausted +[WARN] [1736995172.289570]: Plan exhausted +[WARN] [1736995172.327075]: Plan exhausted +[WARN] [1736995172.377899]: Plan exhausted +[WARN] [1736995172.408598]: Plan exhausted +[INFO] [1736995172.253156]: Video saved! +Starting episode 17 +[INFO] [1736995172.387920]: resetting environment +[INFO] [1736995172.439176]: cleared current plan +[INFO] [1736995172.439785]: moving to home +[INFO] [1736995176.050805]: reached home +[INFO] [1736995176.063032]: reset action history +[INFO] [1736995179.310353]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995185.773174]: resetting environment +[INFO] [1736995185.867321]: cleared current plan +[INFO] [1736995185.884551]: moving to home +[INFO] [1736995186.932567]: reached home +[INFO] [1736995186.969053]: reset action history +[INFO] [1736995190.007776]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_17_2025-01-15-21-39-50.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:37211/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2470166] +process[print_text-53]: started with pid [2470167] +process[rosbag_record-54]: started with pid [2470168] +started bagging! +[INFO] [1736995226.669544]: Storing episode... +[print_text-53] killing on exit +[rosbag_record-54] killing on exit +[pub_text-52] killing on exit +[INFO] [1736995227.510543]: Stored episode 18. +[INFO] [1736995227.538968]: Saving video... +[WARN] [1736995228.018841]: Plan exhausted +[WARN] [1736995228.047992]: Plan exhausted +[WARN] [1736995228.096428]: Plan exhausted +[WARN] [1736995228.132377]: Plan exhausted +[WARN] [1736995228.167185]: Plan exhausted +[WARN] [1736995228.207487]: Plan exhausted +[WARN] [1736995228.251629]: Plan exhausted +[WARN] [1736995228.290540]: Plan exhausted +[WARN] [1736995228.327080]: Plan exhausted +[WARN] [1736995228.370199]: Plan exhausted +[WARN] [1736995228.420115]: Plan exhausted +[WARN] [1736995228.451443]: Plan exhausted +[INFO] [1736995228.228690]: Video saved! +Starting episode 18 +[INFO] [1736995228.396842]: resetting environment +[INFO] [1736995228.455976]: cleared current plan +[INFO] [1736995228.467666]: moving to home +[INFO] [1736995232.094516]: reached home +[INFO] [1736995232.102538]: reset action history +[INFO] [1736995235.330669]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995243.609641]: resetting environment +[INFO] [1736995243.684448]: cleared current plan +[INFO] [1736995243.709667]: moving to home +[INFO] [1736995244.755299]: reached home +[INFO] [1736995244.778225]: reset action history +[INFO] [1736995248.208541]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_18_2025-01-15-21-40-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:46467/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2470309] +process[print_text-56]: started with pid [2470310] +process[rosbag_record-57]: started with pid [2470311] +started bagging! +For topic gripper_width: timestamp difference is 107980243 for nearest: 1736995267053905956 - target: 1736995266945925713 at idx 249 +For topic gripper_width: timestamp difference is 111257150 for nearest: 1736995271053921296 - target: 1736995270942664146 at idx 249 +[INFO] [1736995283.231888]: Storing episode... +[pub_text-55] killing on exit +[print_text-56] killing on exit +[rosbag_record-57] killing on exit +[INFO] [1736995284.066181]: Stored episode 19. +[INFO] [1736995284.079479]: Saving video... +[WARN] [1736995284.580373]: Plan exhausted +[WARN] [1736995284.610761]: Plan exhausted +[WARN] [1736995284.654016]: Plan exhausted +[WARN] [1736995284.720179]: Plan exhausted +[WARN] [1736995284.741994]: Plan exhausted +[WARN] [1736995284.768073]: Plan exhausted +[WARN] [1736995284.815950]: Plan exhausted +[WARN] [1736995284.850855]: Plan exhausted +[WARN] [1736995284.896274]: Plan exhausted +[WARN] [1736995284.926963]: Plan exhausted +[WARN] [1736995284.978772]: Plan exhausted +[WARN] [1736995285.013322]: Plan exhausted +[INFO] [1736995284.754601]: Video saved! +Starting episode 19 +[INFO] [1736995284.958154]: resetting environment +[INFO] [1736995285.040413]: cleared current plan +[INFO] [1736995285.063185]: moving to home +[INFO] [1736995288.809228]: reached home +[INFO] [1736995288.828637]: reset action history +[INFO] [1736995292.058453]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736995295.513791]: resetting environment +[INFO] [1736995295.635542]: cleared current plan +[INFO] [1736995295.661688]: moving to home +[INFO] [1736995296.689978]: reached home +[INFO] [1736995296.713162]: reset action history +[INFO] [1736995300.013875]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/episode_rosbags/episode_19_2025-01-15-21-41-40.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2467097.log +started roslaunch server http://158.130.50.37:44205/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2470453] +process[print_text-59]: started with pid [2470454] +process[rosbag_record-60]: started with pid [2470455] +started bagging! +preempting episode +[INFO] [1736995325.293759]: Storing episode... +[print_text-59] killing on exit +[rosbag_record-60] killing on exit +[pub_text-58] killing on exit +[INFO] [1736995326.221017]: Stored episode 20. +[INFO] [1736995326.249048]: Saving video... +[WARN] [1736995326.610125]: Plan exhausted +[WARN] [1736995326.652779]: Plan exhausted +[WARN] [1736995326.694288]: Plan exhausted +[WARN] [1736995326.729659]: Plan exhausted +[WARN] [1736995326.769265]: Plan exhausted +[WARN] [1736995326.824088]: Plan exhausted +[WARN] [1736995326.847089]: Plan exhausted +[WARN] [1736995326.894136]: Plan exhausted +[WARN] [1736995326.927177]: Plan exhausted +[WARN] [1736995326.968724]: Plan exhausted +[WARN] [1736995327.011495]: Plan exhausted +[WARN] [1736995327.050969]: Plan exhausted +[WARN] [1736995327.110756]: Plan exhausted +[WARN] [1736995327.129713]: Plan exhausted +[WARN] [1736995327.177450]: Plan exhausted +[WARN] [1736995327.209093]: Plan exhausted +[WARN] [1736995327.247444]: Plan exhausted +[WARN] [1736995327.291402]: Plan exhausted +[WARN] [1736995327.345649]: Plan exhausted +[WARN] [1736995327.375617]: Plan exhausted +[WARN] [1736995327.406806]: Plan exhausted +[INFO] [1736995327.021151]: Video saved! +[INFO] [1736995327.323614]: resetting environment +[INFO] [1736995327.422899]: cleared current plan +[INFO] [1736995327.452546]: moving to home +[INFO] [1736995331.113316]: reached home +[INFO] [1736995331.147867]: reset action history +[INFO] [1736995334.583415]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..a480ed1cd3b38df5c4e7ecff218642b42e0995b6 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,91 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..2659f59 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '3456_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..134df65 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,14 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ '3456_0', ++ '3456_1', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d62388e81ac51380dab1e03cf79e3a5ee6974361 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-16T02:25:13.236976", + "startedAt": "2025-01-16T02:25:12.910503", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.49975, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.962, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 302.54129791259766 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d23d4c0d34ddf81cf743839fdaa2a2cae8c834bb --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3d1b728d66af3d385c7669479799a18979a7f2ca22d2f8b5b40e16ebc8b985e7", "size": 1154574, "path": "media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4"}, "global_step": 2369, "_timestamp": 1736995339.6044948, "_runtime": 1026.6858909130096, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "4b58a2f9660a5285887a3d20341f0c589fb20b2547551eaeeae423e564854e87", "size": 1157372, "path": "media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4"}, "eval/num_success": 15.0, "episode": 20.0, "eval/success_rate": 0.75, "eval/success_rate_lb_0.01": 0.4733067452907562, "eval/success_rate_ub_0.01": 0.9187985062599182, "eval/success_rate_lb_0.025": 0.5133531093597412, "eval/success_rate_ub_0.025": 0.8890954852104187, "eval/success_rate_lb_0.05": 0.5846942067146301, "eval/success_rate_ub_0.05": 0.880288302898407, "eval/success_rate_lb_0.1": 0.6031668782234192, "eval/success_rate_ub_0.1": 0.8712705969810486, "eval/2_eval": {"_type": "video-file", "sha256": "bb336114555212728d0bcb3962112f5381481464e1262f66172e00265000a3af", "size": 1151051, "path": "media/videos/eval/2_eval_2_bb336114555212728d0b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "327a1cef6154eb393524a9a4d9524bf91fb1a9c609b08188bf2291d815804422", "size": 1163054, "path": "media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "65dbd3f5364b11e95697bbbe843f808302fe6de1dd8970e11d34656b81a9ce5f", "size": 1162281, "path": "media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "7562c03fef4135eb5f5617db1ec1d73d6999e35e6076fb7fa99289eafa858b90", "size": 753000, "path": "media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "54affceed9218be400c4da12e96bab1b3c4df28d7fe42c121166fe3b49de3b15", "size": 707515, "path": "media/videos/eval/6_eval_6_54affceed9218be400c4.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "8602e79d56991114b037d8b0ab35aa65bc3e1df29aae727eac558f989ea4e07d", "size": 694129, "path": "media/videos/eval/7_eval_7_8602e79d56991114b037.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "2ccbfe16dcecf0f3420f42716100ebe30013ff97399f08262016d1fb3d5f124a", "size": 1174029, "path": "media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "9f85b9e56cea35531fc0f61edc8b99dd9cdb152c2de405de2efc7cab609d4181", "size": 1176285, "path": "media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "be070cda4484f5d8f382748fd4c2f4120827e145f3b8d4e97bd4c7a1afb47ab1", "size": 1184944, "path": "media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "1f892c1875d8f769e839d92205b3decdca4a636b2e461ce5fc10bcb7228b6a03", "size": 1343488, "path": "media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "b7f16c34c55c022ec9cf4d75084ba2f749e4b014ef3b58dd31fb9421f10d545c", "size": 1176474, "path": "media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "b286629d17bfd49bd46bf22225f31e406eb5d7e263ee57e66fb39384533cd8c3", "size": 1033226, "path": "media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "389de3f12b22a271b1c9aadc06c937ff73766ce06d34e3f387c013c7057e77f2", "size": 1185026, "path": "media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "57fbf8f750a05cbd03dd04598b04fce82ad4cc8b09331e81fc130bd77afd9af2", "size": 1393370, "path": "media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "c066e7ff5f521f1bd1c39a3036a4b73720e839fd83caae03a75e0722a74bf27f", "size": 1359276, "path": "media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "8e0251ea6b3f5a1ef81f55f14293484d05071c1988ddd946dfd98d6feb148a3a", "size": 1517963, "path": "media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "792d83e6a625ca80ced859abef65635aa41d06fab9a802b2f960400386a3751f", "size": 1464090, "path": "media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "c347402d83b94d0d76b5e3a7d52f138909cd08d1025f74ebc5b9ca9b69ed8972", "size": 1188193, "path": "media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "c4da0e063c4adfc6cc6679ab62b32593fc7bd21cd854175b8c1ba63217fca8a6", "size": 399, "artifact_path": "wandb-client-artifact://pz9kgffazhuoup17avc81omrq8wwn1kqgoy4dflr9uzchqt36jxudycx7xje3rfezxrfut084xak97oes94ze5revtiu77g52b27d1ukzatbcw8efbcwd42781tj8hht/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ebnix70wfsvbisrrx58rviia36mhc147wxqy6l4wcegideqq31xgd54vhbm14te26k4v1ntjfu4vrm0hli20ca8gt52x2voqi5psf9yk6iozcgwqpj805zb18928is1a:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 15, "total_num_episodes": 20, "total_success_rate": 0.75, "total_success_rate_lb_0.01": 0.4983611702922284, "total_success_rate_ub_0.01": 0.9230575858700546, "total_success_rate_lb_0.025": 0.5425259619866196, "total_success_rate_ub_0.025": 0.8882155864654401, "total_success_rate_lb_0.05": 0.5970279723211784, "total_success_rate_ub_0.05": 0.8681355416038479, "total_success_rate_lb_0.1": 0.585568152350955, "total_success_rate_ub_0.1": 0.862504541801431, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c6c1470740707d609aca41ca00d195ffbc8e3707b9a51c82146a6ac2d6ebb959", "size": 504, "artifact_path": "wandb-client-artifact://d325o9d69g5gj1pp111f0jpx4jvzc357w9dfizrr8fbmjmwsh80xrpdroauf0tm604du6nuandlk3um28l44vshd3n963v6ydef37mf8vcpaytvc6l1sxeqqf498yutq/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://og555wc4ohk8p5knpsn6mllr9j31sm8nq4pkv9uyqpo8wk0uoh9ymc2hzwo1fo2pr1pyriyfvus3sxgoacbe52ykppjfowal3cyl5swm7i55t3o48sm6yd2nhzd5dyh3:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1057}, "eval/total_success_rate": 0.75, "eval/total_success_rate_lb_0.01": 0.49836117029190063, "eval/total_success_rate_ub_0.01": 0.9230575561523438, "eval/total_success_rate_lb_0.025": 0.5425259470939636, "eval/total_success_rate_ub_0.025": 0.8882156014442444, "eval/total_success_rate_lb_0.05": 0.5970279574394226, "eval/total_success_rate_ub_0.05": 0.8681355118751526, "eval/total_success_rate_lb_0.1": 0.5855681300163269, "eval/total_success_rate_ub_0.1": 0.8625045418739319} \ No newline at end of file diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug-internal.log b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d2e2530901a748dc6ba4a612e70c6bdca3e99947 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug-internal.log @@ -0,0 +1,2245 @@ +2025-01-15 21:25:12,918 INFO StreamThr :2467459 [internal.py:wandb_internal():85] W&B internal server running at pid: 2467459, started at: 2025-01-15 21:25:12.918214 +2025-01-15 21:25:12,919 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status +2025-01-15 21:25:12,929 INFO WriterThread:2467459 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb +2025-01-15 21:25:12,929 DEBUG SenderThread:2467459 [sender.py:send():379] send: header +2025-01-15 21:25:12,938 DEBUG SenderThread:2467459 [sender.py:send():379] send: run +2025-01-15 21:25:13,150 INFO SenderThread:2467459 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files +2025-01-15 21:25:13,150 INFO SenderThread:2467459 [sender.py:_start_run_threads():1188] run started: vfeb54wi with start time 1736994312.918604 +2025-01-15 21:25:13,164 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: check_version +2025-01-15 21:25:13,164 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: check_version +2025-01-15 21:25:13,218 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: run_start +2025-01-15 21:25:13,223 DEBUG HandlerThread:2467459 [system_info.py:__init__():26] System info init +2025-01-15 21:25:13,223 DEBUG HandlerThread:2467459 [system_info.py:__init__():41] System info init done +2025-01-15 21:25:13,223 INFO HandlerThread:2467459 [system_monitor.py:start():194] Starting system monitor +2025-01-15 21:25:13,223 INFO SystemMonitor:2467459 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-15 21:25:13,223 INFO HandlerThread:2467459 [system_monitor.py:probe():214] Collecting system info +2025-01-15 21:25:13,224 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started cpu monitoring +2025-01-15 21:25:13,224 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started disk monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started gpu monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started memory monitoring +2025-01-15 21:25:13,225 INFO SystemMonitor:2467459 [interfaces.py:start():188] Started network monitoring +2025-01-15 21:25:13,236 DEBUG HandlerThread:2467459 [system_info.py:probe():152] Probing system +2025-01-15 21:25:13,238 DEBUG HandlerThread:2467459 [system_info.py:_probe_git():137] Probing git +2025-01-15 21:25:13,242 DEBUG HandlerThread:2467459 [system_info.py:_probe_git():145] Probing git done +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_info.py:probe():200] Probing system done +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-16T02:25:13.236976', 'startedAt': '2025-01-16T02:25:12.910503', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.49975, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.962, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 302.54129791259766}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-15 21:25:13,243 INFO HandlerThread:2467459 [system_monitor.py:probe():224] Finished collecting system info +2025-01-15 21:25:13,243 INFO HandlerThread:2467459 [system_monitor.py:probe():227] Publishing system info +2025-01-15 21:25:13,243 DEBUG HandlerThread:2467459 [system_info.py:_save_code():44] Saving code +2025-01-15 21:25:13,244 DEBUG HandlerThread:2467459 [system_info.py:_save_code():67] Saving code done +2025-01-15 21:25:13,244 DEBUG HandlerThread:2467459 [system_info.py:_save_patches():84] Saving git patches +2025-01-15 21:25:13,340 DEBUG HandlerThread:2467459 [system_info.py:_save_patches():126] Saving git patches done +2025-01-15 21:25:13,341 INFO HandlerThread:2467459 [system_monitor.py:probe():229] Finished publishing system info +2025-01-15 21:25:13,344 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-15 21:25:13,344 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-15 21:25:13,345 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-15 21:25:13,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: python_packages +2025-01-15 21:25:13,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: python_packages +2025-01-15 21:25:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:13,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:13,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:13,599 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fdj573ij-wandb-metadata.json +2025-01-15 21:25:13,609 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/jmoc2a32-code/FISH/eval_robot.py +2025-01-15 21:25:13,620 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/346yqiic-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:25:13,623 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/4asr6hxf-diff.patch +2025-01-15 21:25:13,629 INFO HandlerThread:2467459 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/tb" + save: true +} +control { + relay_id: "83dc957e0c7848d8aed4259b6fd3c7e4" +} +_info { + stream_id: "vfeb54wi" +} + +2025-01-15 21:25:13,629 DEBUG HandlerThread:2467459 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-15 21:25:13,666 DEBUG SenderThread:2467459 [sender.py:send():379] send: telemetry +2025-01-15 21:25:13,697 DEBUG SenderThread:2467459 [sender.py:send():379] send: tbrecord +2025-01-15 21:25:13,698 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:25:13,698 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 with policy live +2025-01-15 21:25:13,891 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/6y9bdqf3-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:25:14,151 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH +2025-01-15 21:25:14,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code +2025-01-15 21:25:14,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:15,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:16,151 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:16,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:17,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:18,431 DEBUG SenderThread:2467459 [sender.py:send():379] send: config +2025-01-15 21:25:18,431 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:18,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:19,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:20,152 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:20,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:21,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:22,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:24,153 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:24,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:25,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:26,154 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:26,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:27,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:28,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:28,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:28,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:28,727 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:29,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:30,155 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:25:30,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:31,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:32,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:33,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:34,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:34,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:35,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:36,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:37,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:38,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:39,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:39,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:40,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:41,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:42,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:43,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:43,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:44,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:44,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:45,159 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:25:45,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:46,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:47,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:48,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:49,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:50,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:50,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:51,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:52,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:53,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:54,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:55,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:56,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:25:56,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:57,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:58,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:25:58,583 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:25:58,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:25:59,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:00,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:01,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:01,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:02,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:03,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:04,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:05,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:06,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:07,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:07,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:08,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:09,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:10,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:11,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:12,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:12,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:13,225 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-15 21:26:13,226 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:26:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:13,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:13,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:14,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:15,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:16,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:17,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:17,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:18,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:19,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:20,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:21,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:22,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:23,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:23,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:24,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:25,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:26,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:27,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:28,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:28,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:28,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:28,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:29,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:30,172 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:30,595 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:31,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:32,172 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:32,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:33,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:34,173 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:34,191 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:34,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:35,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:36,173 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:26:36,596 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:37,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:38,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:39,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:39,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:40,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:41,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:42,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:43,227 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:26:43,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:43,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:44,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:44,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:45,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:46,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:47,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:48,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:49,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:50,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:50,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:51,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:52,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:53,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:54,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:55,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:26:55,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:56,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:57,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:58,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:26:58,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:26:58,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:26:58,973 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:26:58,973 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 with policy now +2025-01-15 21:26:58,973 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:26:58,974 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:26:58,974 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:26:58,976 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media +2025-01-15 21:26:59,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos +2025-01-15 21:26:59,209 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/kjtcrf0m-media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,476 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/s0tmh9b8-media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:26:59,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:00,180 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:00,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:00,980 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:01,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:02,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:03,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:04,182 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:04,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:05,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:06,182 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:06,333 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:06,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:07,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:08,183 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:08,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:09,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:10,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:11,588 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:11,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:12,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:13,227 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:27:13,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:13,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:13,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:14,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:15,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:16,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:16,736 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:17,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:18,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:19,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:20,603 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:21,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:22,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:22,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:23,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:24,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:25,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:26,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:27,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:27,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:28,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:28,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:28,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:29,190 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:27:29,450 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/twg1wjqp-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:27:29,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:30,191 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:30,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:31,605 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:32,192 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:32,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:32,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:33,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:34,192 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:34,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:35,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:36,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:37,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:38,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:38,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:39,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:40,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:41,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:42,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:43,228 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:27:43,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:43,584 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:43,680 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:44,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:45,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:46,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:47,607 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:48,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:49,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:49,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:50,198 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:50,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:51,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:52,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:53,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:54,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:27:54,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:55,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:56,200 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:56,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:57,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:58,200 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:27:58,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:27:58,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:27:58,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:59,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:27:59,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:00,201 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:00,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:01,080 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:28:01,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:28:01,080 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 with policy now +2025-01-15 21:28:01,081 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:01,081 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:01,094 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:28:01,306 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/rfbzb3bw-media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,573 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/0x6crfkr-media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:28:01,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:02,202 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:02,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:03,611 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:04,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:04,702 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:05,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:06,204 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:06,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:07,611 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:08,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:09,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:10,205 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:10,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:10,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:11,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:12,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:13,228 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:28:13,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:13,586 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:13,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:14,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:15,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:15,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:16,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:17,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:18,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:19,614 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:20,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:21,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:21,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:22,615 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:23,209 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:28:23,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:24,141 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:24,141 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:24,142 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:24,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:24,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:24,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:25,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:26,210 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:26,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:26,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:27,616 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:28,211 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:28,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:28,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:28,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:29,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:30,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:31,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:32,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:32,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:33,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:34,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:35,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:36,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:37,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:38,501 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:38,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:39,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:40,215 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:40,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:41,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:42,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:43,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:28:43,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:43,585 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:43,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:43,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:44,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:45,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:46,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:47,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:48,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:49,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:49,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:50,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:51,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:52,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:53,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:54,604 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:28:54,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:55,273 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:28:55,273 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 with policy now +2025-01-15 21:28:55,276 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:28:55,277 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:28:55,277 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:28:55,283 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:28:55,490 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/h8kv98eg-media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:55,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:56,220 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:56,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:28:56,513 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/nframv28-media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:28:56,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:57,221 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:28:57,627 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:58,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:28:58,588 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:28:58,633 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:59,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:28:59,961 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:00,222 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:00,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:01,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:02,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:03,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:04,224 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:04,632 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:05,592 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:05,630 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:06,628 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:07,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:08,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:08,978 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:08,978 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:08,978 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:09,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:09,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:09,488 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/t9k3stc7-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:09,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:10,227 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:10,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:10,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:11,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:12,228 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:12,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:13,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:29:13,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:13,586 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:13,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:14,229 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:14,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:15,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:16,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:16,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:17,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:18,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:19,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:20,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:21,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:22,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:22,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:23,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:24,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:25,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:26,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:27,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:27,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:28,586 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:28,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:28,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:29,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:30,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:31,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:32,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:32,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:33,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:34,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:35,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:36,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:37,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:38,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:38,636 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:39,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:40,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:40,647 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:40,966 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:29:40,966 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 with policy now +2025-01-15 21:29:40,970 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:29:40,971 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:40,971 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:40,971 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:41,207 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qxd14u76-media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:41,238 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:29:41,494 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/v035hc8i-media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:29:41,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:42,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:42,653 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:43,229 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:29:43,239 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:43,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:43,588 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:43,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:43,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:44,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:45,668 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:46,240 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:46,657 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:47,654 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:48,241 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:48,660 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:49,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:49,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:50,660 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:51,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:52,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:53,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:54,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:55,032 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:29:55,033 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:29:55,033 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:29:55,034 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:29:55,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:29:55,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:29:55,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:56,243 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:29:56,672 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:57,669 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:58,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:29:58,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:29:58,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:29:59,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:00,245 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:00,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:00,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:01,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:02,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:03,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:04,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:05,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:06,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:06,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:07,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:08,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:09,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:10,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:11,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:11,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:12,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:13,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:30:13,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:13,587 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:13,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:14,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:15,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:16,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:16,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:17,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:18,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:19,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:20,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:21,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:22,598 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:22,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:23,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:24,667 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:25,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:26,242 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:30:26,242 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 with policy now +2025-01-15 21:30:26,245 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:30:26,246 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:30:26,246 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:30:26,246 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:30:26,254 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:26,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:30:26,459 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/wdw28quw-media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:26,674 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:26,725 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/a1s92lgd-media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:30:27,683 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:28,255 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:28,268 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:28,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:28,589 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:28,684 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:29,682 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:30,684 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:31,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:32,256 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:32,685 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:33,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:33,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:34,257 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:34,686 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:35,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:36,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:37,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:38,599 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:38,694 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:39,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:40,088 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:30:40,088 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:30:40,090 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:30:40,259 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:30:40,259 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:30:40,509 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/bziwa5lu-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:30:40,701 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:41,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:42,260 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:42,700 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:43,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:30:43,609 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:43,610 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:43,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:43,694 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:44,260 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:44,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:45,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:46,261 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:46,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:47,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:48,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:49,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:49,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:50,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:51,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:52,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:53,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:54,264 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:30:54,601 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:54,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:55,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:56,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:57,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:58,589 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:30:58,589 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:30:58,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:30:59,669 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:30:59,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:00,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:01,072 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:31:01,072 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 with policy now +2025-01-15 21:31:01,079 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:31:01,079 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:01,079 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:01,080 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:31:01,278 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ckcq6zx5-media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,538 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/1osnfgsm-media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:31:01,696 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:02,267 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:02,697 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:03,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:04,268 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:04,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:04,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:05,701 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:06,268 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:06,702 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:07,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:08,269 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:08,707 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:09,708 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:10,270 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:10,610 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:10,703 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:11,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:12,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:13,231 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:31:13,602 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:13,602 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:13,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:14,721 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:15,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:15,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:16,727 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:17,135 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:17,135 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:17,135 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:17,272 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:17,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:17,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:18,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:18,722 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:19,735 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:20,273 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:20,732 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:20,785 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:21,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:22,274 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:22,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:23,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:24,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:25,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:26,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:26,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:27,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:28,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:28,591 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:28,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:29,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:30,724 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:31,691 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:31,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:32,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:33,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:34,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:35,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:36,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:37,053 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:37,400 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:31:37,400 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 with policy now +2025-01-15 21:31:37,423 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:31:37,423 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:37,423 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:37,424 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:37,632 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/9oj5qj44-media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:37,735 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:38,280 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:31:38,548 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ll3s455q-media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:31:38,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:39,731 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:40,281 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:40,734 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:41,739 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:42,281 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:42,740 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:43,041 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:43,231 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:31:43,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:43,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:43,751 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:44,752 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:45,738 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:46,284 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:46,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:47,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:48,620 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:48,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:49,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:50,769 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:51,179 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:31:51,179 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:31:51,180 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:31:51,285 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:51,286 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:31:51,539 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/nwvjd0p0-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:31:51,755 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:52,286 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:52,760 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:53,764 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:54,276 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:54,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:55,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:56,287 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:31:56,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:57,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:58,591 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:31:58,591 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:31:58,756 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:31:59,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:31:59,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:00,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:01,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:02,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:03,757 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:04,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:05,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:05,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:06,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:07,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:08,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:09,777 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:10,261 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:32:10,261 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 with policy now +2025-01-15 21:32:10,271 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:32:10,271 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:32:10,271 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:32:10,272 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:32:10,292 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:10,292 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:32:10,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:32:10,473 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/b50a1vd9-media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,735 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/yar3cn8d-media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:32:10,771 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:11,305 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:11,774 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:12,293 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:12,783 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:13,232 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:32:13,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:13,600 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:13,774 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:14,773 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:15,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:16,294 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:16,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:16,799 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:17,786 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:18,295 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:18,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:19,788 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:20,787 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:21,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:22,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:22,789 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:23,791 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:24,817 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:25,795 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:26,793 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:27,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:27,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:28,606 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:28,607 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:28,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:29,818 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:30,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:31,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:32,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:32,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:33,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:34,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:35,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:36,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:32:36,235 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:32:36,235 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:32:36,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:32:36,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:32:36,818 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:37,816 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:38,257 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:38,302 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:38,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:39,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:40,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:41,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:42,304 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:42,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:43,232 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:32:43,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:43,629 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:43,629 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:43,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:44,305 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:44,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:45,812 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:46,306 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:32:46,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:47,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:48,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:48,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:49,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:50,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:51,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:52,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:53,620 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:53,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:54,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:55,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:56,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:57,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:58,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:32:58,593 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:32:58,752 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:32:58,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:32:59,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:00,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:01,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:02,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:03,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:04,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:04,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:05,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:06,815 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:07,824 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:08,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:08,740 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:33:08,740 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 with policy now +2025-01-15 21:33:08,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:33:08,758 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:08,758 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:08,759 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:08,823 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:09,052 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qbg4vmhh-media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,314 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:33:09,612 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/lygq9vxg-media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:33:09,803 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:09,829 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:10,315 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:10,826 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:11,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:12,843 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:13,233 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:33:13,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:13,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:13,827 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:14,316 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:14,835 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:15,389 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:15,825 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:16,317 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:16,828 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:17,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:18,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:19,834 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:20,640 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:20,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:21,844 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:22,291 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:22,291 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:22,292 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:22,319 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:22,571 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/h9u60pi3-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:33:22,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:23,842 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:24,320 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:24,862 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:25,686 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:25,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:26,321 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:26,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:27,837 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:28,322 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:28,593 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:28,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:28,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:29,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:30,765 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:30,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:31,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:32,838 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:33,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:34,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:35,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:36,113 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:36,325 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:36,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:37,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:38,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:39,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:40,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:41,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:41,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:42,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:43,233 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:33:43,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:43,594 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:43,839 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:44,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:45,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:46,689 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:46,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:47,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:48,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:49,840 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:50,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:51,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:52,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:52,841 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:53,853 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:54,313 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:33:54,313 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 with policy now +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:33:54,335 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:33:54,336 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:33:54,336 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:33:54,336 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:33:54,544 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/mdztdjn2-media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,811 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/iswk2gaz-media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:33:54,848 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:55,332 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:33:55,854 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:56,333 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:33:56,857 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:57,871 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:57,999 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:33:58,612 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:33:58,612 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:33:58,858 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:33:59,869 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:00,334 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:00,877 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:01,878 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:02,335 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:02,880 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:03,025 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:03,887 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:04,889 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:05,880 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:06,336 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:06,337 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:06,346 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:06,346 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:06,346 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:06,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:07,337 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:07,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:08,338 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:08,679 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:08,913 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:09,899 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:10,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:11,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:12,341 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:12,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:13,234 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:34:13,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:13,597 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:13,741 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:13,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:14,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:15,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:16,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:17,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:18,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:19,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:19,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:20,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:21,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:22,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:23,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:24,650 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:24,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:25,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:26,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:27,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:28,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:28,597 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:28,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:29,758 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:29,893 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:30,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:31,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:32,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:33,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:34,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:35,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:35,894 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:36,895 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:37,930 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:38,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:38,412 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:34:38,412 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 with policy now +2025-01-15 21:34:38,432 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:34:38,432 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:38,432 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:38,433 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:38,634 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/lf60jaaj-media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:38,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:39,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:39,351 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:39,352 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:34:39,630 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/qhot98oe-media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:34:39,907 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:40,352 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:40,656 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:40,933 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:41,941 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:42,944 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:43,234 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:34:43,608 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:43,608 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:43,943 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:44,353 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:44,938 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:45,956 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:46,293 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:46,354 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:46,946 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:47,944 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:48,942 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:49,942 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:50,355 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:50,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:50,401 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:34:50,401 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:34:50,401 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:34:50,605 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ix0sfkor-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:34:50,954 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:51,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:34:51,930 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:51,946 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:52,356 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:52,961 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:53,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:54,965 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:55,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:56,358 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:34:56,977 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:57,045 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:34:57,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:58,621 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:34:58,621 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:34:58,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:34:59,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:00,361 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:00,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:01,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:02,077 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:02,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:03,966 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:04,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:05,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:06,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:07,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:07,967 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:08,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:09,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:10,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:11,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:12,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:12,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:13,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:35:13,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:13,598 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:13,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:14,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:15,968 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:16,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:17,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:17,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:18,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:19,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:20,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:21,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:22,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:23,663 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:23,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:24,969 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:25,971 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:26,371 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:27,026 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:27,203 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:35:27,204 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 with policy now +2025-01-15 21:35:27,216 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:35:27,217 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:35:27,217 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:35:27,217 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:35:27,371 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:35:27,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:27,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:35:27,429 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/kgx5r1rf-media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:27,710 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/d4et035n-media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:35:28,006 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:28,372 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:28,635 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:28,636 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:28,717 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:29,017 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:30,024 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:30,373 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:31,053 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:32,037 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:32,373 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:33,029 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:34,042 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:34,142 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:35,032 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:36,034 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:36,375 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:37,033 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:38,044 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:39,059 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:39,678 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:40,087 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:41,078 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:42,057 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:43,073 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:43,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:35:43,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:35:43,465 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:35:43,465 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:35:43,466 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:35:43,625 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:43,625 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:44,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:44,378 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:35:45,091 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:45,653 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:46,095 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:46,379 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:47,106 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:48,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:48,380 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:49,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:50,080 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:50,381 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:35:50,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:51,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:52,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:53,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:54,081 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:55,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:55,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:35:56,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:57,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:58,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:35:58,617 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:35:58,618 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:35:59,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:00,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:00,764 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:01,082 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:02,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:03,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:04,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:05,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:06,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:06,675 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:07,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:08,083 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:09,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:10,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:10,389 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:11,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:12,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:12,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:13,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:13,235 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:36:13,618 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:13,618 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:14,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:14,390 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:15,084 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:16,104 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:16,391 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:16,591 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:36:16,591 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 with policy now +2025-01-15 21:36:16,600 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:36:16,601 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:36:16,601 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:36:16,601 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:36:16,804 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/i49yfhes-media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,102 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:36:17,663 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/xnhcm2ii-media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:36:17,674 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:18,122 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:18,392 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:19,103 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:20,124 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:21,132 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:22,138 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:22,394 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:23,128 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:23,429 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:24,155 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:24,394 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:25,158 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:26,155 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:27,160 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:28,163 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:28,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:28,648 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:28,761 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:29,147 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:30,149 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:31,160 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:32,167 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:32,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:36:32,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:32,527 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:36:32,527 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:36:32,527 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:36:33,183 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:33,398 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:36:34,179 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:34,399 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:34,725 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:35,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:36,164 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:36,399 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:37,184 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:38,183 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:38,400 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:39,206 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:40,178 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:40,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:41,194 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:42,188 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:43,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:43,236 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:36:43,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:43,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:44,180 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:44,403 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:36:45,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:46,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:46,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:47,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:48,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:49,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:50,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:51,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:51,689 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:52,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:53,181 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:54,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:55,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:56,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:57,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:57,688 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:36:58,182 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:36:58,622 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:36:58,622 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:36:59,198 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:00,014 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:37:00,014 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 with policy now +2025-01-15 21:37:00,025 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:37:00,025 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:00,025 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:00,026 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:00,190 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:00,230 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/q2rr96o6-media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:00,409 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:00,410 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:37:00,677 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/rl4f38nx-media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:37:01,230 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:02,192 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:02,410 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:03,090 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:03,205 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:04,204 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:04,411 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:05,203 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:06,226 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:07,196 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:08,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:08,412 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:08,728 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:09,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:10,219 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:11,210 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:12,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:13,226 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:13,236 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:37:13,575 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:13,576 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:13,576 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:13,656 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:13,656 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:13,746 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:14,267 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:14,415 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:14,666 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/npemfisv-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:15,228 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:16,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:16,416 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:17,228 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:18,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:18,416 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:19,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:19,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:20,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:20,417 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:21,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:22,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:23,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:24,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:24,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:25,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:26,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:27,220 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:28,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:28,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:28,623 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:29,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:29,723 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:30,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:31,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:32,221 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:33,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:33,423 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:34,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:35,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:35,698 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:36,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:37,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:38,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:39,222 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:39,426 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:40,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:40,699 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:41,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:42,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:43,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:43,237 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:37:43,623 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:43,623 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:44,223 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:45,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:45,990 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:46,237 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:46,806 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:37:46,806 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 with policy now +2025-01-15 21:37:46,809 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:37:46,810 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:46,810 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:46,810 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:47,028 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/91eblnjt-media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:47,225 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:47,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:37:47,692 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/xz27t7gr-media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:37:48,261 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:48,430 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:49,254 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:49,431 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:50,282 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:51,261 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:51,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:52,272 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:53,259 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:53,432 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:54,256 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:55,287 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:55,433 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:37:56,292 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:57,009 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:37:57,306 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:58,313 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:58,630 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:37:58,630 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:37:58,631 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:37:58,661 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:37:58,661 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:37:59,329 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:37:59,435 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:37:59,435 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:00,312 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:01,334 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:01,436 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:02,359 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:02,877 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:03,313 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:04,332 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:05,346 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:05,437 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:06,378 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:07,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:08,376 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:08,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:09,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:10,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:11,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:11,440 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:12,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:13,238 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:38:13,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:13,626 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:13,627 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:13,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:14,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:15,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:16,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:17,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:18,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:19,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:19,717 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:20,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:21,348 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:22,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:23,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:24,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:24,718 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:25,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:26,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:27,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:28,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:28,627 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:28,627 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:29,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:29,762 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:30,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:31,349 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:32,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:33,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:34,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:35,350 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:35,719 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:36,357 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:37,358 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:37,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:37,505 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:38:37,505 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:38:37,506 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 with policy now +2025-01-15 21:38:37,506 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:38:37,506 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:38:37,506 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:38:37,770 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ezlyee49-media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:38,392 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:38,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:38,451 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:38,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:38:38,733 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/83pkxwjo-media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:38:39,409 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:39,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:40,380 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:40,452 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:40,737 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:41,380 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:42,373 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:43,238 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:38:43,379 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:43,453 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:43,659 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:43,659 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:44,383 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:45,408 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:46,421 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:46,433 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:47,417 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:47,455 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:48,440 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:49,446 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:50,421 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:51,437 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:51,753 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:52,446 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:53,434 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:54,440 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:54,458 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:38:54,701 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:38:54,701 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:38:54,701 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:38:55,436 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:55,458 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:38:55,459 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:38:56,480 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:57,447 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:57,459 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:38:57,573 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:38:58,470 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:38:58,657 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:38:58,658 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:38:59,454 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:00,465 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:01,449 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:01,461 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:02,493 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:02,750 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:03,464 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:04,460 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:05,460 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:05,463 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:06,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:07,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:07,465 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:08,136 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:08,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:09,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:10,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:11,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:12,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:13,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:39:13,240 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:13,461 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:13,630 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:13,631 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:14,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:15,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:16,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:17,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:17,470 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:18,325 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:18,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:19,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:20,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:21,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:22,462 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:23,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:23,748 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:24,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:25,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:26,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:27,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:28,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:28,631 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:28,631 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:28,793 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:29,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:30,463 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:31,478 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:32,351 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:39:32,351 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 with policy now +2025-01-15 21:39:32,357 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:39:32,358 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:39:32,358 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:39:32,367 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:32,476 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:39:32,477 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:32,576 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/q6owqepi-media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:32,836 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/djti9mvf-media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:39:33,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:33,510 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:34,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:34,523 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:34,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:35,477 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:35,493 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:36,497 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:37,478 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:37,521 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:38,496 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:39,491 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:40,353 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:40,509 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:41,480 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:41,520 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:42,531 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:43,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:39:43,527 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:43,651 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:43,651 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:44,530 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:45,534 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:45,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:45,759 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:39:45,759 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:39:45,759 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:39:46,482 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:39:46,482 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:39:46,524 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:46,759 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ks6w7yv7-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:39:47,483 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:47,530 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:48,558 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:49,483 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:49,577 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:50,550 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:51,116 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:51,484 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:39:51,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:52,561 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:53,564 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:54,594 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:55,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:56,597 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:56,806 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:39:57,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:58,614 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:39:58,652 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:39:58,653 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:39:59,590 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:00,581 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:01,489 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:01,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:01,960 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:02,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:03,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:04,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:05,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:06,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:07,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:07,781 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:08,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:09,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:10,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:11,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:12,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:12,781 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:13,239 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:40:13,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:13,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:13,638 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:14,583 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:15,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:16,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:17,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:17,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:18,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:19,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:20,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:21,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:22,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:22,782 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:23,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:24,584 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:25,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:26,585 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:27,587 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:28,020 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:28,365 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:40:28,365 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 with policy now +2025-01-15 21:40:28,379 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:40:28,380 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:40:28,380 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:40:28,380 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:28,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:40:28,593 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/poyozqqh-media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:28,619 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:28,687 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:28,688 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:28,858 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/o8xocbsf-media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:40:29,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:29,613 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:30,501 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:30,624 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:31,502 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:31,665 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:32,646 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:33,141 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:33,503 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:33,641 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:34,641 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:35,632 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:36,638 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:37,505 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:37,673 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:38,385 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:38,666 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:39,680 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:40,646 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:41,677 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:42,664 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:43,240 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:40:43,415 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:43,507 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:40:43,670 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:43,671 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:43,683 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:43,832 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:40:43,832 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:40:43,833 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:40:44,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:40:44,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:40:44,690 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:45,508 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:45,705 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:46,711 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:47,509 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:47,726 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:48,692 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:48,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:49,510 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:49,700 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:50,710 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:51,511 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:51,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:52,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:53,736 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:53,807 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:40:54,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:55,737 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:56,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:57,513 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:57,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:58,643 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:40:58,643 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:40:58,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:59,514 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:40:59,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:40:59,801 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:00,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:01,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:02,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:03,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:04,712 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:05,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:05,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:06,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:07,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:08,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:09,519 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:09,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:10,713 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:10,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:11,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:12,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:13,240 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:41:13,521 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:13,643 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:13,643 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:13,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:14,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:15,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:16,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:16,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:17,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:18,714 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:19,715 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:20,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:21,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:21,803 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:22,716 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:23,734 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:24,746 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:24,948 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:41:24,948 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 with policy now +2025-01-15 21:41:24,952 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:41:24,952 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:41:24,952 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:41:24,953 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:41:25,181 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fyspcu3h-media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:25,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:41:25,747 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:25,797 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ev5j11ae-media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:41:26,526 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:26,754 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:27,063 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:27,527 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:27,790 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:28,703 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:28,703 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:28,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:29,783 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:30,784 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:31,528 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:31,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:32,070 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:32,800 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:33,529 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:33,779 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:34,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:35,530 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:41:35,804 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:35,898 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:41:35,898 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:41:35,898 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:41:36,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:41:36,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:41:36,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:37,531 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:37,745 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:37,817 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:38,816 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:39,532 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:39,802 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:40,814 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:41,533 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:41,809 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:42,842 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:43,089 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:43,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:41:43,693 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:43,693 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:43,859 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:44,835 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:45,855 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:46,855 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:47,873 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:48,634 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:48,836 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:49,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:50,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:51,538 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:41:51,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:52,831 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:53,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:53,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:54,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:55,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:56,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:57,832 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:58,648 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:41:58,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:41:58,813 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:41:58,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:41:59,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:00,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:01,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:02,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:03,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:04,811 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:04,833 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:05,857 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:06,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:07,277 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:07,277 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 with policy now +2025-01-15 21:42:07,296 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:07,296 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:07,296 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:07,297 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:07,489 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/5qnttwb1-media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:07,545 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval +2025-01-15 21:42:07,806 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/t7o4qhg3-media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:42:07,856 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:08,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:08,886 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:09,546 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:09,850 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:09,858 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:10,875 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:11,859 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:12,873 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:13,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:13,548 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:13,676 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:13,676 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:13,890 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:14,892 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:15,637 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:15,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:16,891 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:17,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:17,909 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:17,947 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:17,948 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:17,948 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:18,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:18,550 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:18,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:19,174 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 21:42:19,174 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 21:42:19,327 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table +2025-01-15 21:42:19,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval +2025-01-15 21:42:19,638 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppigqiw9w +2025-01-15 21:42:19,709 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-15 21:42:19,720 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: partial_history +2025-01-15 21:42:19,767 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/fz6rnj1y-media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:19,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:20,085 INFO SenderThread:2467459 [sender.py:send_request_log_artifact():1525] logged artifact run-vfeb54wi-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MTY2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTEzNDQyMA==', 'latestArtifact': None}} +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:20,085 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json with policy now +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:20,085 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,086 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,086 DEBUG SenderThread:2467459 [sender.py:send():379] send: telemetry +2025-01-15 21:42:20,086 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,090 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,090 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,091 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,091 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,091 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,091 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,092 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,092 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,092 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,092 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,093 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,093 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,093 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,094 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,094 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,094 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,095 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,095 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,095 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,095 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:20,096 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:20,096 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: log_artifact +2025-01-15 21:42:20,334 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ah1sq9kq-media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:42:20,551 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:20,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval +2025-01-15 21:42:20,595 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpknxrse51 +2025-01-15 21:42:20,741 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/vxvgjgrh-media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:20,799 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/ttd8s49d-events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:42:20,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:21,036 INFO SenderThread:2467459 [sender.py:send_request_log_artifact():1525] logged artifact run-vfeb54wi-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MTY3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzOTEzNDQyMQ==', 'latestArtifact': None}} +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send():379] send: files +2025-01-15 21:42:21,037 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:21,037 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json with policy now +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:21,037 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:21,038 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:21,280 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /tmp/tmp_oyimap2wandb/v1krkjej-media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:42:21,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:21,552 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:21,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:22,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:23,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:24,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:25,902 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:26,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:26,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:27,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:28,649 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:28,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:28,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:29,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:30,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:31,823 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:31,918 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:32,557 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:42:32,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:33,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:34,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:35,903 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:36,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:37,820 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:37,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:38,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:39,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:40,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:41,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:42,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:42,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:43,241 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:43,649 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: stop_status +2025-01-15 21:42:43,649 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: stop_status +2025-01-15 21:42:43,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:44,904 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:45,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:46,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:47,821 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:47,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:48,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:49,905 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:42:50,846 DEBUG SenderThread:2467459 [sender.py:send():379] send: exit +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():586] handling exit code: 0 +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():588] handling runtime: 1057 +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:50,846 INFO SenderThread:2467459 [sender.py:send_exit():594] send defer +2025-01-15 21:42:50,846 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,846 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-15 21:42:50,847 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 1 +2025-01-15 21:42:50,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-15 21:42:50,847 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-15 21:42:50,847 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 2 +2025-01-15 21:42:50,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-15 21:42:50,847 INFO HandlerThread:2467459 [system_monitor.py:finish():203] Stopping system monitor +2025-01-15 21:42:50,847 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-15 21:42:50,847 DEBUG SystemMonitor:2467459 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-15 21:42:50,849 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined cpu monitor +2025-01-15 21:42:50,849 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined disk monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined gpu monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined memory monitor +2025-01-15 21:42:50,913 INFO HandlerThread:2467459 [interfaces.py:finish():200] Joined network monitor +2025-01-15 21:42:50,914 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,914 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-15 21:42:50,914 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 3 +2025-01-15 21:42:50,914 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,914 DEBUG SenderThread:2467459 [sender.py:send():379] send: stats +2025-01-15 21:42:50,914 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-15 21:42:50,915 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:50,915 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-15 21:42:50,915 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 4 +2025-01-15 21:42:50,916 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:50,916 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-15 21:42:51,567 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:53,568 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:56,958 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: keepalive +2025-01-15 21:42:56,959 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,959 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-15 21:42:56,959 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 5 +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send():379] send: history +2025-01-15 21:42:56,960 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: summary_record +2025-01-15 21:42:56,960 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,960 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-15 21:42:56,961 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:56,961 DEBUG SenderThread:2467459 [sender.py:send():379] send: summary +2025-01-15 21:42:56,961 INFO SenderThread:2467459 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 6 +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,962 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 7 +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: status_report +2025-01-15 21:42:56,962 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:56,962 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-15 21:42:56,962 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:56,962 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-15 21:42:57,570 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:42:57,847 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:42:59,953 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 8 +2025-01-15 21:42:59,953 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:42:59,954 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:59,954 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-15 21:42:59,954 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:42:59,954 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-15 21:42:59,954 INFO SenderThread:2467459 [job_builder.py:build():440] Attempting to build job artifact +2025-01-15 21:42:59,957 INFO SenderThread:2467459 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-15 21:42:59,959 INFO SenderThread:2467459 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-15 21:42:59,976 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 9 +2025-01-15 21:42:59,976 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:42:59,976 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-15 21:42:59,976 DEBUG SenderThread:2467459 [sender.py:send():379] send: artifact +2025-01-15 21:43:00,572 INFO Thread-12 :2467459 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:43:00,626 INFO wandb-upload_0:2467459 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphw5sjkl5 +2025-01-15 21:43:00,848 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:00,891 INFO wandb-upload_2:2467459 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk_xp6b81 +2025-01-15 21:43:00,941 INFO wandb-upload_3:2467459 [upload_job.py:push():88] Uploaded file /tmp/tmpx6bsw9uf/wandb-job.json +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0NDA3MjMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-15 21:43:01,577 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-15 21:43:01,577 INFO SenderThread:2467459 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml config.yaml +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json wandb-summary.json +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 events.out.tfevents.1736994313.leonmkim-ROG-Strix-G15CS-G15CS.2467097.0 +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt requirements.txt +2025-01-15 21:43:02,573 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-metadata.json wandb-metadata.json +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log output.log +2025-01-15 21:43:02,574 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/diff.patch diff.patch +2025-01-15 21:43:02,575 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 media/videos/eval/15_eval_15_57fbf8f750a05cbd03dd.mp4 +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 media/videos/eval/11_eval_11_1f892c1875d8f769e839.mp4 +2025-01-15 21:43:02,576 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 media/videos/eval/5_eval_5_7562c03fef4135eb5f56.mp4 +2025-01-15 21:43:02,577 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 media/videos/eval/19_eval_19_c347402d83b94d0d76b5.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 media/videos/eval/17_eval_17_8e0251ea6b3f5a1ef81f.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 media/videos/eval/14_eval_14_389de3f12b22a271b1c9.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 media/videos/eval/6_eval_6_54affceed9218be400c4.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 media/videos/eval/3_eval_3_327a1cef6154eb393524.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 media/videos/eval/2_eval_2_bb336114555212728d0b.mp4 +2025-01-15 21:43:02,580 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 media/videos/eval/12_eval_12_b7f16c34c55c022ec9cf.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 media/videos/eval/16_eval_16_c066e7ff5f521f1bd1c3.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 media/videos/eval/13_eval_13_b286629d17bfd49bd46b.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 media/videos/eval/18_eval_18_792d83e6a625ca80ced8.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 media/videos/eval/1_eval_1_4b58a2f9660a5285887a.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 media/videos/eval/9_eval_9_9f85b9e56cea35531fc0.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 media/videos/eval/10_eval_10_be070cda4484f5d8f382.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 media/videos/eval/4_eval_4_65dbd3f5364b11e95697.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 media/videos/eval/0_eval_0_3d1b728d66af3d385c76.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 media/videos/eval/8_eval_8_2ccbfe16dcecf0f3420f.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 media/videos/eval/7_eval_7_8602e79d56991114b037.mp4 +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json media/table/eval/total_success_rate_ci_21_c6c1470740707d609aca.table.json +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json media/table/eval/success_rate_ci_20_c4da0e063c4adfc6cc66.table.json +2025-01-15 21:43:02,581 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 10 +2025-01-15 21:43:02,582 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:02,582 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,582 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-15 21:43:02,582 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,582 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-15 21:43:02,582 INFO SenderThread:2467459 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 21:43:02,756 INFO wandb-upload_0:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/wandb-summary.json +2025-01-15 21:43:02,759 INFO wandb-upload_2:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/requirements.txt +2025-01-15 21:43:02,760 INFO wandb-upload_1:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/config.yaml +2025-01-15 21:43:02,789 INFO wandb-upload_3:2467459 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/files/output.log +2025-01-15 21:43:02,849 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:02,849 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:02,989 INFO Thread-11 (_thread_body):2467459 [sender.py:transition_state():614] send defer: 11 +2025-01-15 21:43:02,989 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,989 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-15 21:43:02,989 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,990 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-15 21:43:02,990 INFO SenderThread:2467459 [file_pusher.py:join():175] waiting for file pusher +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 12 +2025-01-15 21:43:02,993 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:02,993 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-15 21:43:02,993 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-15 21:43:02,993 INFO SenderThread:2467459 [file_stream.py:finish():601] file stream finish called +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [file_stream.py:finish():605] file stream finish is done +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 13 +2025-01-15 21:43:03,229 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:03,229 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-15 21:43:03,229 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-15 21:43:03,229 INFO SenderThread:2467459 [sender.py:transition_state():614] send defer: 14 +2025-01-15 21:43:03,229 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: defer +2025-01-15 21:43:03,230 INFO HandlerThread:2467459 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: final +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send():379] send: footer +2025-01-15 21:43:03,230 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: defer +2025-01-15 21:43:03,230 INFO SenderThread:2467459 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-15 21:43:03,230 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:03,231 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: server_info +2025-01-15 21:43:03,231 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:03,231 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: server_info +2025-01-15 21:43:03,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-15 21:43:03,235 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: get_summary +2025-01-15 21:43:03,236 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-15 21:43:03,237 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-15 21:43:03,295 DEBUG SenderThread:2467459 [sender.py:send_request():406] send_request: poll_exit +2025-01-15 21:43:03,296 INFO MainThread:2467459 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-15 21:43:03,296 INFO MainThread:2467459 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-15 21:43:03,297 INFO MainThread:2467459 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-15 21:43:03,297 DEBUG HandlerThread:2467459 [handler.py:handle_request():158] handle_request: shutdown +2025-01-15 21:43:03,297 INFO HandlerThread:2467459 [handler.py:finish():882] shutting down handler +2025-01-15 21:43:04,235 INFO WriterThread:2467459 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [sender.py:finish():1615] shutting down sender +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [file_pusher.py:finish():169] shutting down file pusher +2025-01-15 21:43:04,296 INFO SenderThread:2467459 [file_pusher.py:join():175] waiting for file pusher diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug.log b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c1b8673697e695a086b02d4bda70ce40bafd4a37 --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Configure stats pid to 2467097 +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/settings +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-15 21:25:12,912 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug.log +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/logs/debug-internal.log +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():569] calling init triggers +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_0', 'wandb_notes': '3456_0_req_3921_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi'} +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():619] starting backend +2025-01-15 21:25:12,913 INFO MainThread:2467097 [wandb_init.py:init():623] setting up manager +2025-01-15 21:25:12,917 INFO MainThread:2467097 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-15 21:25:12,918 INFO MainThread:2467097 [wandb_init.py:init():631] backend started and connected +2025-01-15 21:25:12,928 INFO MainThread:2467097 [wandb_init.py:init():720] updated telemetry +2025-01-15 21:25:12,936 INFO MainThread:2467097 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-15 21:25:13,164 INFO MainThread:2467097 [wandb_run.py:_on_init():2435] communicating current version +2025-01-15 21:25:13,213 INFO MainThread:2467097 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.3 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-15 21:25:13,213 INFO MainThread:2467097 [wandb_init.py:init():804] starting run threads in backend +2025-01-15 21:25:13,583 INFO MainThread:2467097 [wandb_run.py:_console_start():2413] atexit reg +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-15 21:25:13,584 INFO MainThread:2467097 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-15 21:25:13,586 INFO MainThread:2467097 [wandb_init.py:init():847] run started, returning control to user process +2025-01-15 21:25:13,586 INFO MainThread:2467097 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_0/vfeb54wi/tb, True +2025-01-15 21:25:18,430 INFO MainThread:2467097 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-15 21:43:04,298 WARNING MsgRouterThr:2467097 [router.py:message_loop():77] message_loop has been closed diff --git a/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ac3a14315e217de7ce4fdc88e2d0c2aa1c9954fb --- /dev/null +++ b/vfeb54wi/wandb/run-20250115_212512-vfeb54wi/run-vfeb54wi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:099c6ef30de08a8204316790c9a286e2d411dda7f441ca90af3511937145a17f +size 243417