diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..9ff86c2603d67833687ad3557c9242edccd59d73 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,74 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_1_2025-01-07-23-10-30.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_5_2025-01-07-23-13-42.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_16_2025-01-07-23-23-26.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_2_2025-01-07-23-11-16.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_18_2025-01-07-23-25-15.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_10_2025-01-07-23-18-32.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_6_2025-01-07-23-14-26.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_19_2025-01-07-23-26-12.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1evqtpqc/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +you6ychq/episode_rosbags/episode_3_2025-01-07-17-43-50.bag filter=lfs diff=lfs merge=lfs -text +you6ychq/episode_rosbags/episode_2_2025-01-07-17-43-04.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_15_2025-01-07-23-22-42.bag filter=lfs diff=lfs merge=lfs -text +you6ychq/episode_rosbags/episode_4_2025-01-07-17-44-32.bag filter=lfs diff=lfs merge=lfs -text +you6ychq/episode_rosbags/episode_0_2025-01-07-17-41-33.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_13_2025-01-07-23-20-57.bag filter=lfs diff=lfs merge=lfs -text +1evqtpqc/episode_rosbags/episode_14_2025-01-07-23-21-48.bag filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 filter=lfs diff=lfs merge=lfs -text +yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/1evqtpqc/.hydra/config.yaml b/1evqtpqc/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a56df077cebbbb54879d37edcc83ff7d1e7c8eb --- /dev/null +++ b/1evqtpqc/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/1evqtpqc/.hydra/hydra.yaml b/1evqtpqc/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54121df7adb1b3c1ef5863b50a6723899628c66e --- /dev/null +++ b/1evqtpqc/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/1evqtpqc/.hydra/overrides.yaml b/1evqtpqc/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/1evqtpqc/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/1evqtpqc/episode_rosbags/aligned_depth_to_color_K.npy b/1evqtpqc/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1evqtpqc/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1evqtpqc/episode_rosbags/cam_tf_world.npy b/1evqtpqc/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/1evqtpqc/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/1evqtpqc/episode_rosbags/color_K.npy b/1evqtpqc/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1evqtpqc/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1evqtpqc/episode_rosbags/depth_K.npy b/1evqtpqc/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/1evqtpqc/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/1evqtpqc/episode_rosbags/episode_10_2025-01-07-23-18-32.bag b/1evqtpqc/episode_rosbags/episode_10_2025-01-07-23-18-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..54e35e741ecf9f36278f739523ac74668603d54e --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_10_2025-01-07-23-18-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a3277dd69dd411ceaf3d7caa1694f748786c00b06204663be28f313685d65b +size 1414938177 diff --git a/1evqtpqc/episode_rosbags/episode_13_2025-01-07-23-20-57.bag b/1evqtpqc/episode_rosbags/episode_13_2025-01-07-23-20-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..196580b7bf730b4070b945d75af408672c4d2668 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_13_2025-01-07-23-20-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8d42eed2ce1c3c9b4cf84b8700f2446f82459f0fd2d36bd05fe53d714ac1c9 +size 1410567301 diff --git a/1evqtpqc/episode_rosbags/episode_14_2025-01-07-23-21-48.bag b/1evqtpqc/episode_rosbags/episode_14_2025-01-07-23-21-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..8506a4697efba389a30cd71dcde1860867f89627 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_14_2025-01-07-23-21-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:136d87af06a002c2880a61b22731ac78ea4875a8ee1d7edea36ec1762a86fe66 +size 1409518342 diff --git a/1evqtpqc/episode_rosbags/episode_15_2025-01-07-23-22-42.bag b/1evqtpqc/episode_rosbags/episode_15_2025-01-07-23-22-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..fd5a097a92534d2df4adbdb92bbef6c675fbfcf6 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_15_2025-01-07-23-22-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1e8ce6fd1e4a735df6a38c6fea076ffee79421d89855903f27cc50660109777 +size 1410794632 diff --git a/1evqtpqc/episode_rosbags/episode_16_2025-01-07-23-23-26.bag b/1evqtpqc/episode_rosbags/episode_16_2025-01-07-23-23-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..63425c92dd94b03cde390ffd345403d7adb55117 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_16_2025-01-07-23-23-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2671934695c717021d6e95133b9ee800b989e05ad002928360ad6f663986200 +size 1412784608 diff --git a/1evqtpqc/episode_rosbags/episode_18_2025-01-07-23-25-15.bag b/1evqtpqc/episode_rosbags/episode_18_2025-01-07-23-25-15.bag new file mode 100644 index 0000000000000000000000000000000000000000..c01a2617b8ea998f9f3ff0efca2f1c389bdd140f --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_18_2025-01-07-23-25-15.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7950ce63a8ba9248473650f8d0fc5a3b944dc82fdcb9899473ce374fe0dca05c +size 1408817240 diff --git a/1evqtpqc/episode_rosbags/episode_19_2025-01-07-23-26-12.bag b/1evqtpqc/episode_rosbags/episode_19_2025-01-07-23-26-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..721e7b3f5acbc09b921ee5c2d56aa72aca0132e9 --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_19_2025-01-07-23-26-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d1b273aaa1eb7c070cd39f9a36a41fb4f3c92d0f397f5a8cb65307ae971606 +size 1409971005 diff --git a/1evqtpqc/episode_rosbags/episode_1_2025-01-07-23-10-30.bag b/1evqtpqc/episode_rosbags/episode_1_2025-01-07-23-10-30.bag new file mode 100644 index 0000000000000000000000000000000000000000..7e51e6597c6d7e99ab57cbe0d88c0dfcae168b9e --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_1_2025-01-07-23-10-30.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad6548aceee3c472ffa19e6a156d2500a94ff6de1060e9f03b71297abefdbe65 +size 1411535492 diff --git a/1evqtpqc/episode_rosbags/episode_2_2025-01-07-23-11-16.bag b/1evqtpqc/episode_rosbags/episode_2_2025-01-07-23-11-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..d404424c795a8f61f0f3ac3b5a3ca56f1c2e97fd --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_2_2025-01-07-23-11-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ee8a615ce06052a488fe45dfa91f886225c44df516f1987e48f3a708056b97 +size 1412686962 diff --git a/1evqtpqc/episode_rosbags/episode_5_2025-01-07-23-13-42.bag b/1evqtpqc/episode_rosbags/episode_5_2025-01-07-23-13-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..791c585976cd6e9fa97f621f0b686313ff86170a --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_5_2025-01-07-23-13-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5824b4be8aa63e0eba0f07a5257e8eead6417a78d254a92cca5d730ab3f1ff8a +size 1413041503 diff --git a/1evqtpqc/episode_rosbags/episode_6_2025-01-07-23-14-26.bag b/1evqtpqc/episode_rosbags/episode_6_2025-01-07-23-14-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..d6e754cd5e465ff174bcb6473cacd08d7203aeca --- /dev/null +++ b/1evqtpqc/episode_rosbags/episode_6_2025-01-07-23-14-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:176ef5878fe0428f5913a3eb4557bc3e125a6d9b8686ebfc136d32531dad2b72 +size 1409277532 diff --git a/1evqtpqc/eval_robot.log b/1evqtpqc/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..98a2df1151981ff13ebf2c3aafe67d7f401fa979 --- /dev/null +++ b/1evqtpqc/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 23:09:09,507][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 23:09:09,510][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/1evqtpqc/eval_video/0_eval.mp4 b/1evqtpqc/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbaa1b664c542a700658e803110e0907f9d1c35d --- /dev/null +++ b/1evqtpqc/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5d8645307c977c2178ad77a7431d6a1bfe84d48f087236fd6a919884f4fd31 +size 1123217 diff --git a/1evqtpqc/eval_video/10_eval.mp4 b/1evqtpqc/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7d9867fb04d3fe289ff02b9d7d55f02978dd7a6 --- /dev/null +++ b/1evqtpqc/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b875e5dd12f721236870e25266e34d93e52e8918f9f496a9c963163a64d2489e +size 1152134 diff --git a/1evqtpqc/eval_video/11_eval.mp4 b/1evqtpqc/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fdbd2b77dce1d78cfdad6b5e9be9dbdccb57ffc9 --- /dev/null +++ b/1evqtpqc/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3e8fea09465a60cb8c5b576f7b7880e547ee963d3ed592e68d2993e233d3c1f +size 1337182 diff --git a/1evqtpqc/eval_video/12_eval.mp4 b/1evqtpqc/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77507d1a9849cc98c440255d4acad586eed3e08a --- /dev/null +++ b/1evqtpqc/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84f1b7d03de03c92ec2c8a41dcf82819dd1d23781bc8a7035a2d0186430c5e46 +size 1441497 diff --git a/1evqtpqc/eval_video/13_eval.mp4 b/1evqtpqc/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4dc40f033501af565b468a1db742c598dd8694a8 --- /dev/null +++ b/1evqtpqc/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f4d6e4d981133c66e545ed42f83f6c0a6bcfd59bcd2dd381b8098c8bdb468f +size 1416179 diff --git a/1evqtpqc/eval_video/14_eval.mp4 b/1evqtpqc/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3623568dac6e9868a21e0d2975f0579a69e6656 --- /dev/null +++ b/1evqtpqc/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e423f85e531e4aa2eda45108b3dc3234ecca077473ebbbcf9d7c7a637943ec73 +size 1366701 diff --git a/1evqtpqc/eval_video/15_eval.mp4 b/1evqtpqc/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01ba6168a0f7ac3618526f33f24c68ba56121a41 --- /dev/null +++ b/1evqtpqc/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b8c887cc79fe4bb5aafaaa9f836cc32e9fd296371b4b7b141de76c7f939d76e +size 1156265 diff --git a/1evqtpqc/eval_video/16_eval.mp4 b/1evqtpqc/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3560386dcf0cc6f85e5081acbb1b7b405738b4dc --- /dev/null +++ b/1evqtpqc/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01858989d758192b59d620577b9d6cc12c4e50516030cc566a2832125470ad02 +size 1473520 diff --git a/1evqtpqc/eval_video/17_eval.mp4 b/1evqtpqc/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7281d6584e41cdf7d9ab5dd10e264e19464c5352 --- /dev/null +++ b/1evqtpqc/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c3c2ba281fd3f6a37339e1c72ad37f093ad6d75a9aebbf31fdd389f59f0f35b +size 1503945 diff --git a/1evqtpqc/eval_video/18_eval.mp4 b/1evqtpqc/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..070fc5b8ff8e7f3b41d93a13fc52353c8d470118 --- /dev/null +++ b/1evqtpqc/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d9fd71a9a5e8f25cb3e878d51a4389410db4b2a906bdb1efe163abb52a82fb9 +size 1579185 diff --git a/1evqtpqc/eval_video/19_eval.mp4 b/1evqtpqc/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18a1a7c433689df3be0fb3f49fbd5a1c6ed194e5 --- /dev/null +++ b/1evqtpqc/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49104c03ad255c9ff4c035a0d9465a2b6ee3ae5ec1832113e329373a9f7e4029 +size 1494448 diff --git a/1evqtpqc/eval_video/1_eval.mp4 b/1evqtpqc/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76b162adf754768f796b556b57a46c4f79267491 --- /dev/null +++ b/1evqtpqc/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6cba8accd20e1b8ec9e4ca9adb5f90d53eedbfed622cbc55026038c3eeb3bf4 +size 1140024 diff --git a/1evqtpqc/eval_video/2_eval.mp4 b/1evqtpqc/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbe290e877439fb90cfb5fc516399be17bb7f6c9 --- /dev/null +++ b/1evqtpqc/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b724e203bcdc67aa5a2f04fd8853c3feb4a28749a40adf615618fd51313c8103 +size 1138364 diff --git a/1evqtpqc/eval_video/3_eval.mp4 b/1evqtpqc/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b016c2fd5b2d08f24840385b5fb3c032ce7b9da2 --- /dev/null +++ b/1evqtpqc/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a1f2288b71555e1168140547170669ba0fd7488f4b222591ceed7f4ba76eb2f +size 1148121 diff --git a/1evqtpqc/eval_video/4_eval.mp4 b/1evqtpqc/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37ade52c0a86d59ee8223b59269f72904f533414 --- /dev/null +++ b/1evqtpqc/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642bdc6d95c526c17f0731df62950ff1c8a72317347fe989e1beacec0f890218 +size 1136660 diff --git a/1evqtpqc/eval_video/5_eval.mp4 b/1evqtpqc/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d3eb5baff409f733e759871efd4ee2ec7b776787 --- /dev/null +++ b/1evqtpqc/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:465c60fb8be6cb25f820c9dcc462df06f973d03ee40f2b5968e1b91d4016af54 +size 1152499 diff --git a/1evqtpqc/eval_video/6_eval.mp4 b/1evqtpqc/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14f4b601ef9781614417077b78524193b309f517 --- /dev/null +++ b/1evqtpqc/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df7000ad25a0ae7e07a93d59d2ef5951cde4a989edd3763a5830c026b03493ea +size 1136363 diff --git a/1evqtpqc/eval_video/7_eval.mp4 b/1evqtpqc/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ba73b581407be07aaac3e7226f31d9bf3cbd8d9 --- /dev/null +++ b/1evqtpqc/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6581231703b0f374d0369ade51de91bb5ff149d5e9481b5d2e6b4fc2a3e94e +size 1227690 diff --git a/1evqtpqc/eval_video/8_eval.mp4 b/1evqtpqc/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..874cba567242ae8707c3b9cf0eb67d9a29e3ae6a --- /dev/null +++ b/1evqtpqc/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b915a87c852375007f764e85b3f4d657656758a473755f307c82a18f9cf15b3 +size 1357981 diff --git a/1evqtpqc/eval_video/9_eval.mp4 b/1evqtpqc/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29ca282b6234ab9ebbb0920776b60c2e8603d5fc --- /dev/null +++ b/1evqtpqc/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:811955fe3a96368a3ead687220eb14337c378c6e9d92710b9de75b248a2d3166 +size 1181724 diff --git a/1evqtpqc/tb/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 b/1evqtpqc/tb/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 new file mode 100644 index 0000000000000000000000000000000000000000..f0d4c329ee981408daf9210e3c97bf222c5faf64 --- /dev/null +++ b/1evqtpqc/tb/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a279f704b17c3eac4e1c74afd96b1876c64d2af7cd088e8a26fc7a5beea1ba3 +size 22784 diff --git a/1evqtpqc/wandb/debug-internal.log b/1evqtpqc/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..18f572813b9faa97530846fd5ef897514a3eb249 --- /dev/null +++ b/1evqtpqc/wandb/debug-internal.log @@ -0,0 +1,2274 @@ +2025-01-07 23:09:15,294 INFO StreamThr :2249564 [internal.py:wandb_internal():85] W&B internal server running at pid: 2249564, started at: 2025-01-07 23:09:15.294013 +2025-01-07 23:09:15,295 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status +2025-01-07 23:09:15,307 INFO WriterThread:2249564 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb +2025-01-07 23:09:15,308 DEBUG SenderThread:2249564 [sender.py:send():379] send: header +2025-01-07 23:09:15,316 DEBUG SenderThread:2249564 [sender.py:send():379] send: run +2025-01-07 23:09:15,559 INFO SenderThread:2249564 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files +2025-01-07 23:09:15,559 INFO SenderThread:2249564 [sender.py:_start_run_threads():1188] run started: 1evqtpqc with start time 1736309355.294227 +2025-01-07 23:09:15,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 23:09:15,578 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: check_version +2025-01-07 23:09:15,623 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 23:09:15,628 DEBUG HandlerThread:2249564 [system_info.py:__init__():26] System info init +2025-01-07 23:09:15,628 DEBUG HandlerThread:2249564 [system_info.py:__init__():41] System info init done +2025-01-07 23:09:15,628 INFO HandlerThread:2249564 [system_monitor.py:start():194] Starting system monitor +2025-01-07 23:09:15,628 INFO SystemMonitor:2249564 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 23:09:15,629 INFO HandlerThread:2249564 [system_monitor.py:probe():214] Collecting system info +2025-01-07 23:09:15,629 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 23:09:15,629 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started disk monitoring +2025-01-07 23:09:15,630 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 23:09:15,631 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started memory monitoring +2025-01-07 23:09:15,631 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started network monitoring +2025-01-07 23:09:15,641 DEBUG HandlerThread:2249564 [system_info.py:probe():152] Probing system +2025-01-07 23:09:15,642 DEBUG HandlerThread:2249564 [system_info.py:_probe_git():137] Probing git +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:_probe_git():145] Probing git done +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:probe():200] Probing system done +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T04:09:15.641385', 'startedAt': '2025-01-08T04:09:15.286608', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.159125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.081, 'min': 800.0, 'max': 4700.0}, {'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9039688110352}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 23:09:15,647 INFO HandlerThread:2249564 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 23:09:15,647 INFO HandlerThread:2249564 [system_monitor.py:probe():227] Publishing system info +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:_save_code():44] Saving code +2025-01-07 23:09:15,649 DEBUG HandlerThread:2249564 [system_info.py:_save_code():67] Saving code done +2025-01-07 23:09:15,649 DEBUG HandlerThread:2249564 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 23:09:15,710 DEBUG HandlerThread:2249564 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 23:09:15,710 INFO HandlerThread:2249564 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 23:09:15,713 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 23:09:15,941 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 23:09:15,941 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: python_packages +2025-01-07 23:09:15,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:15,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:15,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:15,977 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ldm9zc23-wandb-metadata.json +2025-01-07 23:09:15,985 INFO HandlerThread:2249564 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/tb" + save: true +} +control { + relay_id: "0e59b649b6e646d485666f1a9a652098" +} +_info { + stream_id: "1evqtpqc" +} + +2025-01-07 23:09:15,985 DEBUG HandlerThread:2249564 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 23:09:16,013 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rtz14662-diff.patch +2025-01-07 23:09:16,031 DEBUG SenderThread:2249564 [sender.py:send():379] send: telemetry +2025-01-07 23:09:16,035 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/erpa07wq-code/FISH/eval_robot.py +2025-01-07 23:09:16,036 DEBUG SenderThread:2249564 [sender.py:send():379] send: tbrecord +2025-01-07 23:09:16,037 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:09:16,037 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 with policy live +2025-01-07 23:09:16,242 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/chj417hk-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code +2025-01-07 23:09:16,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH +2025-01-07 23:09:16,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:17,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:18,560 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:18,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:19,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:20,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:20,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:21,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:22,788 DEBUG SenderThread:2249564 [sender.py:send():379] send: config +2025-01-07 23:09:22,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:23,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:24,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:24,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:25,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:25,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:26,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:26,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:27,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:28,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:28,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:29,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:30,564 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:30,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:30,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:30,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:31,015 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:31,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:32,564 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:32,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:33,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:34,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:35,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:36,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:36,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:37,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:38,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:39,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:40,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:41,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:41,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:42,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:43,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:44,567 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:44,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:45,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:45,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:45,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:46,568 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:46,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:46,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:47,568 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:09:47,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:48,569 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:48,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:49,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:50,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:51,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:52,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:52,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:53,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:54,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:55,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:56,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:57,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:57,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:58,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:59,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:00,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:00,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:00,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:01,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:02,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:03,021 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:03,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:04,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:05,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:06,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:07,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:08,879 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:08,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:09,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:10,574 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:10,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:11,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:12,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:13,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:14,418 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:14,575 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:14,624 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:10:14,624 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 with policy now +2025-01-07 23:10:14,625 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:10:14,626 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:10:14,626 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:10:14,626 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:10:14,912 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/7hyyubkf-media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:14,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:10:15,631 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 23:10:15,632 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:10:15,874 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/y7kph9t8-media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:15,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:15,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:15,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:16,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:16,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:17,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:18,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:19,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:20,277 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:20,578 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:20,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:21,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:22,578 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:22,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:23,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:24,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:25,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:25,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:26,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:27,580 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:10:27,853 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/uabotbvt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:10:27,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:28,580 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:28,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:29,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:30,581 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:30,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:30,947 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:30,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:30,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:31,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:32,582 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:32,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:33,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:34,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:35,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:36,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:36,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:37,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:38,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:39,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:40,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:41,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:41,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:42,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:43,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:44,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:45,633 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:10:45,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:45,945 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:45,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:46,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:47,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:47,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:48,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:49,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:50,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:51,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:52,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:52,588 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:52,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:53,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:54,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:55,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:56,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:57,472 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:57,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:58,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:58,730 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:10:58,730 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 with policy now +2025-01-07 23:10:58,733 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:10:58,733 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:10:58,733 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:10:58,734 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:10:58,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:58,987 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/c1usdr64-media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:10:59,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,591 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:10:59,892 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/zladk6ho-media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:00,591 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:00,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:00,950 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:00,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:01,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:02,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:03,421 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:03,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:04,592 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:04,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:05,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:06,593 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:06,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:07,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:08,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:08,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:09,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:10,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:11,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:12,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:13,399 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:11:13,399 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:11:13,400 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:11:13,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:11:13,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:11:13,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:14,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:14,697 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:14,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:15,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:11:15,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:15,948 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:15,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:16,596 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:16,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:17,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:18,597 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:18,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:19,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:20,631 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:20,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:21,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:22,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:23,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:24,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:25,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:25,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:26,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:27,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:28,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:29,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:30,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:30,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:30,948 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:30,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:31,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:32,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:33,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:34,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:35,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:35,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:36,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:37,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:38,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:39,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:40,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:40,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:41,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:42,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:43,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:44,605 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:44,920 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:11:44,920 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 with policy now +2025-01-07 23:11:44,921 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:11:44,921 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:11:44,921 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:11:44,922 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:11:44,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:45,173 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/a0bqb8rd-media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:11:45,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:11:45,899 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/gvum62ui-media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:45,951 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:45,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:46,037 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:46,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:46,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:47,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:47,975 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:48,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:49,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:50,607 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:50,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:51,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:51,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:52,608 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:52,982 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:53,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:54,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:55,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:56,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:56,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:57,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:58,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:59,982 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:00,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:00,953 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:00,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:01,238 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:01,238 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:01,239 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:01,611 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:01,611 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:01,897 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/382o6bh8-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:01,983 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:02,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:02,612 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:02,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:03,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:04,612 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:04,988 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:05,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:06,613 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:06,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:07,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:07,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:08,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:09,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:10,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:11,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:12,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:13,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:13,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:14,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:15,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:12:15,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:15,951 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:15,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:16,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:17,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:18,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:19,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:19,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:20,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:21,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:22,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:23,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:24,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:24,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:25,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:26,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:27,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:28,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:29,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:30,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:30,952 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:30,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:31,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:31,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:32,998 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:33,665 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:12:33,665 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 with policy now +2025-01-07 23:12:33,668 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:12:33,668 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:33,668 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:33,670 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:33,957 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/imyk92rb-media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:33,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:34,622 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:34,622 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:34,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:34,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:12:34,921 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/49axge9l-media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:35,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:35,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:35,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:36,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:37,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:38,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:38,624 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:38,998 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:39,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:40,999 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:41,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:42,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:42,625 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:43,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:44,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:45,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:45,635 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:12:45,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:45,964 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:46,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:47,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:47,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:48,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:49,011 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:49,627 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:50,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:50,295 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:50,296 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:50,296 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:50,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:50,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:51,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:52,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:52,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:52,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:53,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:54,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:54,629 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:55,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:56,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:57,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:58,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:58,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:59,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:00,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:00,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:00,952 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:01,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:02,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:03,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:04,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:04,102 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:05,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:06,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:07,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:08,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:09,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:09,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:10,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:11,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:12,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:13,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:14,041 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:15,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:15,041 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:15,635 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:13:15,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:15,958 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:16,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:16,637 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:17,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:18,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:19,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:20,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:20,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:21,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:21,701 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:13:21,701 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 with policy now +2025-01-07 23:13:21,705 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:13:21,705 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:13:21,705 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:13:21,706 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:13:21,953 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/mzs6vtga-media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:22,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:22,639 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:13:22,639 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:22,640 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:22,640 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:13:22,944 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/s6gkztjj-media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:23,020 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:24,018 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:25,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:25,347 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:26,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:26,641 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:27,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:28,033 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:29,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:30,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:30,642 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:30,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:30,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:31,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:31,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:32,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:33,029 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:34,046 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:35,043 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:36,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:36,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:37,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:38,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:38,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:13:38,895 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/7dbonkd8-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:13:39,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:39,355 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:13:39,356 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:13:39,356 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:13:39,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:13:40,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:40,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:41,052 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:41,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:42,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:43,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:44,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:44,647 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:45,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:45,636 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:13:45,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:45,957 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:46,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:47,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:47,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:48,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:49,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:50,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:51,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:52,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:52,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:53,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:54,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:54,651 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:55,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:56,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:57,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:57,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:58,067 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:59,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:00,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:00,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:00,957 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:01,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:02,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:03,031 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:03,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:04,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:05,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:06,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:07,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:08,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:08,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:09,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:10,075 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:10,569 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:14:10,570 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 with policy now +2025-01-07 23:14:10,581 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:14:10,581 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:10,582 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:10,582 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:10,657 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:14:10,847 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/t0l6xdru-media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:11,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:11,123 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/iffolz4h-media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:12,086 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:12,657 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:13,087 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:13,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:14,092 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:15,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:15,636 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:14:15,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:15,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:16,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:16,658 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:17,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:18,101 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:18,659 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:19,117 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:19,169 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:20,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:21,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:22,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:22,410 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:22,410 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:22,410 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:22,660 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:22,660 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:14:23,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:24,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:24,269 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:24,661 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:25,117 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:26,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:27,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:28,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:28,663 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:29,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:29,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:30,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:30,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:30,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:31,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:32,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:33,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:34,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:34,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:35,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:36,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:36,665 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:37,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:38,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:38,666 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:39,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:40,074 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:40,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:41,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:42,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:43,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:44,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:45,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:45,637 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:14:45,638 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:45,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:45,958 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:46,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:47,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:48,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:49,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:50,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:50,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:51,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:52,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:53,116 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:54,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:54,531 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:14:54,531 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 with policy now +2025-01-07 23:14:54,535 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:14:54,535 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:54,535 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:54,536 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:14:54,785 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/mvotjo30-media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:55,045 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/bnkkep60-media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:55,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:55,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:56,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:56,673 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:57,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:57,673 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:58,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:59,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:00,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:00,674 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:00,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:00,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:01,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:01,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:02,137 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:02,675 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:03,135 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:04,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:05,148 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:06,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:06,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:07,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:07,463 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:15:07,464 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:15:07,464 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:15:07,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:15:07,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:15:07,930 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ki06q0ks-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:15:08,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:08,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:09,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:10,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:10,678 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:11,169 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:12,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:12,678 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:12,895 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:13,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:14,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:15,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:15,637 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:15:15,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:15,960 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:16,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:17,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:18,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:18,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:19,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:20,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:21,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:22,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:23,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:23,984 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:24,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:25,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:26,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:27,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:28,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:29,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:29,984 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:30,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:30,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:30,961 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:31,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:32,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:33,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:34,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:35,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:35,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:36,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:37,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:38,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:39,168 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:39,989 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:15:39,989 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 with policy now +2025-01-07 23:15:39,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:15:39,992 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:15:39,992 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:15:39,992 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:15:40,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:40,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:40,247 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/vowh0va8-media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:40,688 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:15:40,974 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/m6a9zk1j-media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:41,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:41,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:42,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:42,690 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:43,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:44,184 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:44,690 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:45,187 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:45,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:45,638 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:15:45,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:45,980 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:46,189 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:47,182 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:48,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:48,692 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:49,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:50,202 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:50,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:51,198 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:52,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:53,194 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:54,210 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:55,214 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:55,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:56,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:57,216 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:58,219 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:59,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:00,217 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:00,977 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:00,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:01,064 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:01,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:02,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:03,224 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:04,221 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:05,223 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:06,223 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:07,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:07,226 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:08,230 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:09,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:10,232 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:11,236 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:12,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:12,233 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:13,238 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:14,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:15,244 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:15,638 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:16:15,970 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:15,970 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:16,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:17,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:17,243 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:18,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:19,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:20,270 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:21,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:22,255 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:23,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:23,264 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:24,258 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:25,265 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:26,268 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:27,263 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:28,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:28,259 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:29,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:30,279 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:30,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:30,966 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:31,276 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:32,281 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:33,077 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:33,274 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:34,275 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:35,280 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:36,283 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:37,283 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:38,284 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:39,022 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:39,280 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:40,281 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:41,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:42,302 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:43,300 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:44,025 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:44,311 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:45,303 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:45,639 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:16:45,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:45,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:46,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:47,298 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:48,317 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:49,083 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:49,326 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:50,349 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:50,592 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:16:50,592 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:16:50,593 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:16:50,714 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:16:50,714 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:16:51,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:52,310 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:52,715 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:16:53,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:54,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:54,327 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:55,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:56,324 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:56,716 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:16:57,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:58,328 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:59,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:59,658 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:00,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:00,718 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:00,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:00,963 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:01,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:02,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:03,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:04,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:04,719 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:04,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:05,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:06,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:07,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:08,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:08,721 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:09,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:10,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:10,641 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:11,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:12,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:13,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:14,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:15,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:15,639 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:17:15,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:15,964 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:16,046 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:16,337 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:17,334 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:18,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:19,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:20,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:21,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:22,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:22,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:23,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:24,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:25,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:26,593 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:17:26,593 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 with policy now +2025-01-07 23:17:26,600 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:17:26,601 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:17:26,602 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:17:26,602 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:17:26,731 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:17:26,856 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/g2209p0a-media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:27,131 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/thr302eg-media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:27,360 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:27,616 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:28,368 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:28,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:29,370 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:30,377 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:30,733 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:30,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:30,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:31,363 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:32,371 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:33,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:33,363 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:34,380 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:34,736 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:35,368 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:36,390 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:37,372 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:38,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:38,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:39,390 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:40,373 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:41,391 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:42,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:42,652 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:17:42,652 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:17:42,653 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:17:43,002 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/d1jpx36j-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:17:43,387 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:43,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:44,388 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:44,740 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:45,388 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:45,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:17:45,969 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:45,969 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:46,395 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:46,741 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:47,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:48,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:48,742 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:49,073 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:49,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:50,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:51,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:52,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:53,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:54,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:55,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:55,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:56,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:57,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:58,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:59,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:00,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:00,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:00,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:00,966 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:01,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:02,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:03,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:04,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:05,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:05,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:06,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:07,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:08,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:09,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:10,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:11,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:11,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:12,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:13,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:14,351 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:18:14,351 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 with policy now +2025-01-07 23:18:14,360 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:18:14,360 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:18:14,360 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:18:14,361 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:18:14,400 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:14,608 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/twgpqrl9-media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:18:15,028 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rzsrf72v-media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:15,404 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:15,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:18:15,972 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:15,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:16,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:16,399 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:16,753 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:17,399 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:18,426 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:18,753 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:19,423 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:20,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:21,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:22,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:22,424 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:22,755 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:23,427 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:24,425 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:25,428 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:26,440 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:27,052 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:27,428 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:28,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:28,706 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:18:28,707 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:18:28,707 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:18:28,757 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:18:28,757 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:28,758 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:18:29,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:30,478 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:30,758 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:30,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:30,978 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:31,444 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:32,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:32,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:32,759 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:33,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:34,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:34,760 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:35,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:36,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:37,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:37,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:38,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:39,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:40,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:41,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:42,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:43,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:43,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:44,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:45,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:45,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:18:45,972 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:45,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:46,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:47,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:48,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:48,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:49,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:50,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:51,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:52,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:53,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:54,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:54,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:55,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:56,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:57,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:58,434 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:59,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:59,444 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:00,436 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:00,505 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:19:00,505 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 with policy now +2025-01-07 23:19:00,505 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:19:00,506 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:00,506 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:00,506 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:00,767 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/s89wjrvk-media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:19:00,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:00,980 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:01,038 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/yb54cmhj-media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:01,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:01,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:02,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:02,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:03,451 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:04,129 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:04,452 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:04,771 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:05,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:06,468 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:06,772 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:07,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:08,469 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:08,773 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:09,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:09,486 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:10,495 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:11,488 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:12,473 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:13,492 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:13,760 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:13,761 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:13,762 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:13,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:13,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:19:14,025 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/r81g15bt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:19:14,500 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:14,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:15,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:15,492 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:15,641 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:19:15,983 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:15,983 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:16,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:17,500 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:18,507 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:18,777 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:19,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:20,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:20,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:21,537 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:22,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:22,779 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:23,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:24,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:24,779 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:25,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:25,583 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:26,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:27,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:28,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:29,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:30,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:30,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:30,974 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:31,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:31,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:32,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:33,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:34,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:35,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:36,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:37,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:37,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:38,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:39,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:40,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:41,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:42,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:42,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:43,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:44,523 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:45,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:45,641 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:19:45,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:45,974 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:46,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:47,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:48,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:48,522 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:49,524 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:50,532 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:50,621 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:19:50,621 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 with policy now +2025-01-07 23:19:50,661 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:19:50,661 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:50,662 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:50,662 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:50,789 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:50,789 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:50,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:50,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:19:50,872 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/bphbgl47-media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:51,143 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/eeo2xvy0-media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:51,530 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:52,541 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:52,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:53,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:53,552 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:53,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:54,551 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:55,556 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:56,543 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:56,792 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:57,542 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:58,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:58,557 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:58,792 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:59,571 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:00,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:01,000 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:01,000 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:01,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:02,562 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:02,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:02,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:20:02,818 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:02,818 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:02,818 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:03,497 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:03,555 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:03,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:04,573 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:04,795 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:05,564 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:06,574 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:07,569 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:08,534 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:08,562 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:08,796 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:09,570 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:10,582 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:11,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:12,582 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:13,586 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:13,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:14,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:14,799 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:15,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:15,642 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:20:15,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:15,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:16,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:17,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:18,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:19,105 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:19,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:20,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:21,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:22,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:23,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:24,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:25,082 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:25,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:26,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:27,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:28,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:29,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:30,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:30,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:30,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:31,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:31,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:32,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:33,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:34,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:35,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:36,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:37,083 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:37,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:38,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:39,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:40,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:41,611 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:41,869 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:20:41,870 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 with policy now +2025-01-07 23:20:41,878 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:20:41,879 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:41,879 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:41,879 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:42,130 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/17k0r1ko-media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:42,606 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:42,811 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:20:42,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:43,146 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/h61tw7jt-media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:43,612 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:43,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:44,602 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:45,604 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:45,642 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:20:46,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:46,008 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:46,607 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:46,812 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:47,609 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:48,099 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:48,605 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:49,627 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:50,643 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:50,813 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:51,642 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:52,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:53,143 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:53,631 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:53,814 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:20:53,881 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:53,881 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:53,881 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:54,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:54,815 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:54,815 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:55,632 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:56,665 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:57,636 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:58,274 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:58,638 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:58,817 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:59,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:00,648 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:01,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:01,013 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:01,671 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:02,667 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:03,411 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:03,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:04,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:04,819 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:05,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:06,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:07,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:08,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:08,821 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:09,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:09,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:10,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:11,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:12,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:13,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:14,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:14,824 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:14,877 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:15,643 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:21:15,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:15,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:15,990 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:16,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:16,825 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:17,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:18,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:19,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:20,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:20,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:20,827 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:21,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:22,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:23,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:24,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:25,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:26,100 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:26,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:27,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:28,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:29,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:30,668 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:30,836 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:31,025 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:31,025 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:31,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:31,669 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:31,969 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:21:31,969 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 with policy now +2025-01-07 23:21:31,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:21:31,985 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:21:31,986 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:21:31,986 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:21:32,226 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/3b41yzzt-media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:32,704 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:32,838 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:21:33,133 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/w7hak22a-media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:33,677 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:33,838 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:34,677 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:35,698 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:36,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:36,686 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:36,839 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:37,694 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:38,682 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:39,734 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:40,699 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:40,840 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:41,141 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:41,702 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:42,718 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:43,731 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:44,732 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:44,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:44,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:21:44,943 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:21:44,943 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:21:44,943 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:21:45,104 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/03z4z2vt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:21:45,643 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:21:45,740 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:45,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:21:46,044 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:46,044 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:46,206 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:46,732 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:46,843 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:47,746 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:48,728 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:49,753 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:50,746 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:50,844 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:51,738 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:52,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:52,760 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:53,748 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:54,748 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:54,846 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:55,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:56,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:56,848 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:57,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:58,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:58,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:59,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:00,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:01,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:01,002 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:01,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:02,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:03,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:03,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:04,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:05,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:06,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:07,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:08,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:08,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:09,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:10,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:11,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:12,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:13,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:13,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:14,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:15,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:22:15,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:16,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:16,002 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:16,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:17,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:18,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:18,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:19,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:20,753 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:21,770 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:22,764 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:22,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:23,015 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:22:23,015 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 with policy now +2025-01-07 23:22:23,033 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:22:23,034 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:22:23,034 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:22:23,034 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:22:23,283 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/do2pu68c-media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:23,757 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:22:24,120 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:24,159 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ny7cwklp-media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:24,798 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:24,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:25,772 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:25,860 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:26,777 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:27,766 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:28,792 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:28,861 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:29,777 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:29,786 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:30,781 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:30,862 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:31,020 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:31,020 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:31,789 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:32,808 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:33,797 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:34,791 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:35,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:35,809 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:36,807 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:37,810 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:38,005 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:22:38,005 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:22:38,006 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:22:38,820 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:22:39,817 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:40,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:40,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:40,867 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:41,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:42,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:42,868 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:43,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:44,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:44,868 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:45,302 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:45,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:22:45,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:46,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:46,006 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:46,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:47,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:48,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:49,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:50,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:51,112 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:51,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:52,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:53,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:54,817 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:55,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:56,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:57,112 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:57,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:58,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:59,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:00,833 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:01,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:01,006 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:01,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:02,136 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:02,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:03,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:04,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:05,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:06,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:07,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:08,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:08,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:09,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:10,874 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:10,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:11,463 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:23:11,463 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 with policy now +2025-01-07 23:23:11,490 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:23:11,490 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:23:11,491 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:23:11,491 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:23:11,712 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/dd30u4x4-media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:11,851 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:11,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:23:12,157 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/dgu28mn5-media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:12,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:12,887 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:13,595 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:13,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:13,885 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:14,865 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:15,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:23:15,888 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:16,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:16,026 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:16,882 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:16,900 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:17,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:18,874 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:18,883 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:19,501 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:19,891 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:20,884 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:20,891 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:21,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:22,878 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:22,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:22,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:23:23,063 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:23:23,063 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:23:23,064 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:23:23,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:23:23,905 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:24,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:24,886 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:24,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:25,900 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:26,898 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:27,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:28,887 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:28,906 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:29,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:29,913 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:30,908 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:31,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:31,057 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:31,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:32,932 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:33,941 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:34,934 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:35,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:35,916 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:36,891 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:36,916 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:37,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:38,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:39,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:40,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:41,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:41,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:42,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:43,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:44,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:45,645 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:23:45,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:46,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:46,013 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:46,166 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:46,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:47,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:48,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:49,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:50,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:51,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:52,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:52,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:53,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:54,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:55,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:56,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:57,132 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:57,899 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:57,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:58,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:59,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:00,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:01,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:01,015 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:01,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:02,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:02,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:03,964 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:24:03,964 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 with policy now +2025-01-07 23:24:03,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:04,021 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:24:04,022 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:24:04,022 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:24:04,022 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:24:04,233 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/u2zwnre1-media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:04,911 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:24:04,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:04,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:24:04,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:05,223 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/51cglfgr-media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:05,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:05,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:06,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:07,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:07,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:08,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:09,913 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:09,995 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:10,986 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:11,999 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:12,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:13,142 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:13,915 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:14,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:15,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:15,646 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:24:15,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:16,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:16,065 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:17,011 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:18,018 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:18,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:18,917 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:24:19,016 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:19,132 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:24:19,132 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:24:19,132 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:24:19,189 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ye62zc23-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:24:19,918 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:24:19,918 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:20,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:21,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:21,919 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:22,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:23,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:23,785 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:24,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:25,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:25,920 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:26,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:27,051 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:28,063 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:29,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:29,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:30,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:31,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:31,057 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:31,105 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:32,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:33,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:33,924 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:34,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:34,835 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:35,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:36,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:37,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:38,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:39,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:40,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:40,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:41,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:41,928 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:42,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:43,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:44,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:45,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:45,165 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:45,646 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:24:46,031 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:46,031 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:46,077 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:47,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:48,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:49,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:50,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:50,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:51,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:52,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:53,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:54,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:55,037 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:56,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:56,162 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:57,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:58,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:59,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:59,942 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:00,038 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:25:00,038 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 with policy now +2025-01-07 23:25:00,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:25:00,043 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:00,043 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:00,044 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:00,078 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:00,307 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/1s2u1h5y-media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:00,942 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:00,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:00,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:25:01,051 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:01,051 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:01,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:01,162 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:01,239 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/io1ylyl0-media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:01,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:02,067 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:03,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:04,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:05,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:05,944 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:06,096 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:06,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:07,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:08,073 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:09,085 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:09,946 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:10,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:11,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:11,176 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:11,194 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:11,194 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:11,194 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:11,947 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:11,947 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:25:12,128 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:13,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:13,948 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:14,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:15,121 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:15,647 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:25:16,044 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:16,044 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:16,098 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:16,180 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:17,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:17,950 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:18,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:19,119 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:20,102 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:21,119 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:22,144 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:22,175 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:23,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:24,132 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:25,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:26,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:27,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:27,954 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:28,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:28,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:29,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:29,955 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:30,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:31,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:31,034 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:31,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:32,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:33,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:33,669 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:33,957 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:34,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:35,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:36,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:37,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:38,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:39,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:39,176 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:40,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:41,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:42,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:43,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:44,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:45,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:45,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:45,647 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:25:46,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:46,035 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:46,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:47,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:48,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:49,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:50,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:50,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:51,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:52,126 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:53,139 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:53,966 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:54,140 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:54,341 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:25:54,341 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 with policy now +2025-01-07 23:25:54,365 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:25:54,366 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:54,366 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:54,366 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:54,606 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/d4r27cpb-media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:25:55,148 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:55,287 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/m7ke0vqm-media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:55,452 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:55,968 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:56,151 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:57,175 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:58,173 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:59,163 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:59,969 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:00,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:00,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:01,078 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:01,078 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:01,177 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:02,187 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:03,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:03,971 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:04,205 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:05,189 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:06,207 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:06,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:07,205 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:07,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:26:08,212 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:08,266 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:26:08,266 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:26:08,266 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:26:08,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:26:08,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:26:09,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:09,974 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:10,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:11,221 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:11,241 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:11,974 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:12,227 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:13,243 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:13,975 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:14,272 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:15,248 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:15,648 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:26:16,075 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:16,075 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:16,231 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:16,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:17,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:18,271 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:19,265 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:20,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:21,239 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:21,285 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:22,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:23,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:23,980 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:24,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:25,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:26,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:27,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:27,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:28,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:29,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:30,258 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:31,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:31,050 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:31,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:32,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:32,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:33,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:34,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:35,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:36,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:37,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:37,733 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:37,986 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:38,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:39,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:40,254 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:41,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:42,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:42,930 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:43,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:43,989 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:44,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:45,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:45,648 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:26:46,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:46,050 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:46,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:47,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:48,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:48,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:49,310 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:49,991 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:50,276 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:50,296 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:26:50,296 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 with policy now +2025-01-07 23:26:50,341 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:26:50,341 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:26:50,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:26:50,342 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:26:50,566 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/da34jd6f-media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:26:51,277 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:51,293 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/tp4pa9fo-media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:51,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:52,284 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:53,217 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:53,278 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:54,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:55,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:55,994 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:56,351 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:57,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:58,298 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:59,215 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:59,338 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:59,996 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:00,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:01,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:27:01,066 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:27:01,328 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:01,329 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:01,329 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:01,330 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:01,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:01,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:02,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:02,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:27:02,629 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:27:02,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:27:02,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table +2025-01-07 23:27:03,224 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/8ue5b6ml-media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:03,312 INFO wandb-upload_0:2249564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6msy2ivh +2025-01-07 23:27:03,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:03,476 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:27:03,486 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:27:03,711 INFO SenderThread:2249564 [sender.py:send_request_log_artifact():1525] logged artifact run-1evqtpqc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjY0MjUxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkyNzQzOA==', 'latestArtifact': None}} +2025-01-07 23:27:03,711 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:27:03,711 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json with policy now +2025-01-07 23:27:03,712 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:03,712 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,713 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send():379] send: telemetry +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,713 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,715 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,715 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,716 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,716 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,716 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,717 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,717 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,717 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,717 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,718 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,718 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,718 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,719 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,719 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,719 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,719 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,720 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,720 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,720 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,720 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:27:03,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:04,000 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval +2025-01-07 23:27:04,002 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/x9oxhdwl-media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:04,234 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/at5tb6y0-media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:04,282 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/vhynu0g9-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:04,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:04,518 INFO wandb-upload_0:2249564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1xjl6479 +2025-01-07 23:27:04,898 INFO SenderThread:2249564 [sender.py:send_request_log_artifact():1525] logged artifact run-1evqtpqc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjY0MjUyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkyNzQ0MQ==', 'latestArtifact': None}} +2025-01-07 23:27:04,901 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:05,013 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:27:05,014 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json with policy now +2025-01-07 23:27:05,014 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:05,014 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:05,014 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:05,291 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rg32mlkp-media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:05,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:06,017 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:06,017 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:06,018 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:27:06,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:07,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:08,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:09,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:10,220 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:10,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:11,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:12,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:13,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:14,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:15,220 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:15,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:15,649 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:27:16,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:27:16,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:27:16,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:17,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:18,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:19,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:20,053 DEBUG SenderThread:2249564 [sender.py:send():379] send: exit +2025-01-07 23:27:20,053 INFO SenderThread:2249564 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_exit():588] handling runtime: 1084 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_exit():594] send defer +2025-01-07 23:27:20,054 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,054 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 23:27:20,054 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 1 +2025-01-07 23:27:20,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 23:27:20,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,055 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 23:27:20,055 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 2 +2025-01-07 23:27:20,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 23:27:20,055 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 23:27:20,055 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 23:27:20,056 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 23:27:20,107 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 23:27:20,108 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 23:27:20,108 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined network monitor +2025-01-07 23:27:20,108 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,108 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 23:27:20,108 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 3 +2025-01-07 23:27:20,108 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:27:20,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,109 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 23:27:20,109 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,109 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 23:27:20,109 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 4 +2025-01-07 23:27:20,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,109 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 23:27:21,025 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:22,025 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 23:27:26,341 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,341 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,342 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 23:27:26,342 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 5 +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:26,342 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,342 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:26,343 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:26,343 DEBUG SenderThread:2249564 [sender.py:send():379] send: summary +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:26,344 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 6 +2025-01-07 23:27:26,344 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,344 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 23:27:26,344 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 23:27:26,345 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 7 +2025-01-07 23:27:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,345 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 23:27:26,345 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,345 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 23:27:27,028 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:27,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:27,613 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 8 +2025-01-07 23:27:27,613 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:27,614 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:27,614 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 23:27:27,614 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:27,614 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 23:27:27,614 INFO SenderThread:2249564 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 23:27:27,616 INFO SenderThread:2249564 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 23:27:27,618 INFO SenderThread:2249564 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 23:27:27,629 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 9 +2025-01-07 23:27:27,630 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:27,630 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 23:27:27,630 DEBUG SenderThread:2249564 [sender.py:send():379] send: artifact +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 23:27:27,885 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 23:27:28,028 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml config.yaml +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json wandb-summary.json +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt requirements.txt +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json wandb-metadata.json +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log output.log +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch diff.patch +2025-01-07 23:27:28,031 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 10 +2025-01-07 23:27:28,039 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,039 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 23:27:28,039 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:27:28,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,214 INFO wandb-upload_3:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:27:28,227 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:28,243 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:28,252 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt +2025-01-07 23:27:28,453 INFO Thread-11 (_thread_body):2249564 [sender.py:transition_state():614] send defer: 11 +2025-01-07 23:27:28,453 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,453 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 23:27:28,453 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,453 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 23:27:28,453 INFO SenderThread:2249564 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 23:27:28,456 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 12 +2025-01-07 23:27:28,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,457 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 23:27:28,457 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,457 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 23:27:28,457 INFO SenderThread:2249564 [file_stream.py:finish():601] file stream finish called +2025-01-07 23:27:28,655 INFO SenderThread:2249564 [file_stream.py:finish():605] file stream finish is done +2025-01-07 23:27:28,655 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 13 +2025-01-07 23:27:28,655 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,655 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 14 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send():379] send: final +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send():379] send: footer +2025-01-07 23:27:28,656 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,656 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,657 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,658 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: server_info +2025-01-07 23:27:28,658 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 23:27:28,660 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 23:27:28,661 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:28,708 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,708 INFO MainThread:2249564 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 23:27:28,709 INFO MainThread:2249564 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 23:27:28,709 INFO MainThread:2249564 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 23:27:28,709 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 23:27:28,709 INFO HandlerThread:2249564 [handler.py:finish():882] shutting down handler +2025-01-07 23:27:29,658 INFO WriterThread:2249564 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [sender.py:finish():1615] shutting down sender +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [file_pusher.py:join():175] waiting for file pusher diff --git a/1evqtpqc/wandb/debug.log b/1evqtpqc/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..35e82fc403ed099ce68e5c2e109f7b2d2507964f --- /dev/null +++ b/1evqtpqc/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Configure stats pid to 2249399 +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/settings +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug.log +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug-internal.log +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():569] calling init triggers +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc'} +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():619] starting backend +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():623] setting up manager +2025-01-07 23:09:15,293 INFO MainThread:2249399 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 23:09:15,293 INFO MainThread:2249399 [wandb_init.py:init():631] backend started and connected +2025-01-07 23:09:15,307 INFO MainThread:2249399 [wandb_init.py:init():720] updated telemetry +2025-01-07 23:09:15,313 INFO MainThread:2249399 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 23:09:15,578 INFO MainThread:2249399 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 23:09:15,618 INFO MainThread:2249399 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 23:09:15,618 INFO MainThread:2249399 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 23:09:15,944 INFO MainThread:2249399 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 23:09:15,945 INFO MainThread:2249399 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 23:09:15,945 INFO MainThread:2249399 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/tb, True +2025-01-07 23:09:22,787 INFO MainThread:2249399 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodin'} +2025-01-07 23:27:29,710 WARNING MsgRouterThr:2249399 [router.py:message_loop():77] message_loop has been closed diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9256995565e91a8f8425d71a513a62fefc3380bc --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736309355 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodin diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3f33d68d5f61ef69d5b4133841e12781c858f071 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch @@ -0,0 +1,30 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..0b377ae 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json new file mode 100644 index 0000000000000000000000000000000000000000..fd531a01931dde846bb325c4eaabc70ecba620ff --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.55, 0.28449782733458906, 0.7815009653005266, 0.31899209324165356, 0.737304761958345, 0.3815420940755294, 0.7210296019469707, 0.40086738394693344, 0.7047718874700606]]} \ No newline at end of file diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9ccf28be3f63ba3f4672b9890fa00fc280dc012c --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[58, 165, 0.3515151515151515, 0.2702030539972229, 0.4412672668812916, 0.28281411532931056, 0.42427372933948604, 0.29532181475679786, 0.41212901847331795, 0.30248962160424797, 0.4012609683174675]]} \ No newline at end of file diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbaa1b664c542a700658e803110e0907f9d1c35d --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5d8645307c977c2178ad77a7431d6a1bfe84d48f087236fd6a919884f4fd31 +size 1123217 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b7d9867fb04d3fe289ff02b9d7d55f02978dd7a6 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b875e5dd12f721236870e25266e34d93e52e8918f9f496a9c963163a64d2489e +size 1152134 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fdbd2b77dce1d78cfdad6b5e9be9dbdccb57ffc9 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3e8fea09465a60cb8c5b576f7b7880e547ee963d3ed592e68d2993e233d3c1f +size 1337182 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77507d1a9849cc98c440255d4acad586eed3e08a --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84f1b7d03de03c92ec2c8a41dcf82819dd1d23781bc8a7035a2d0186430c5e46 +size 1441497 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4dc40f033501af565b468a1db742c598dd8694a8 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f4d6e4d981133c66e545ed42f83f6c0a6bcfd59bcd2dd381b8098c8bdb468f +size 1416179 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3623568dac6e9868a21e0d2975f0579a69e6656 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e423f85e531e4aa2eda45108b3dc3234ecca077473ebbbcf9d7c7a637943ec73 +size 1366701 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01ba6168a0f7ac3618526f33f24c68ba56121a41 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b8c887cc79fe4bb5aafaaa9f836cc32e9fd296371b4b7b141de76c7f939d76e +size 1156265 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3560386dcf0cc6f85e5081acbb1b7b405738b4dc --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01858989d758192b59d620577b9d6cc12c4e50516030cc566a2832125470ad02 +size 1473520 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7281d6584e41cdf7d9ab5dd10e264e19464c5352 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c3c2ba281fd3f6a37339e1c72ad37f093ad6d75a9aebbf31fdd389f59f0f35b +size 1503945 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..070fc5b8ff8e7f3b41d93a13fc52353c8d470118 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d9fd71a9a5e8f25cb3e878d51a4389410db4b2a906bdb1efe163abb52a82fb9 +size 1579185 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..18a1a7c433689df3be0fb3f49fbd5a1c6ed194e5 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49104c03ad255c9ff4c035a0d9465a2b6ee3ae5ec1832113e329373a9f7e4029 +size 1494448 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76b162adf754768f796b556b57a46c4f79267491 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6cba8accd20e1b8ec9e4ca9adb5f90d53eedbfed622cbc55026038c3eeb3bf4 +size 1140024 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cbe290e877439fb90cfb5fc516399be17bb7f6c9 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b724e203bcdc67aa5a2f04fd8853c3feb4a28749a40adf615618fd51313c8103 +size 1138364 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b016c2fd5b2d08f24840385b5fb3c032ce7b9da2 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a1f2288b71555e1168140547170669ba0fd7488f4b222591ceed7f4ba76eb2f +size 1148121 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37ade52c0a86d59ee8223b59269f72904f533414 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:642bdc6d95c526c17f0731df62950ff1c8a72317347fe989e1beacec0f890218 +size 1136660 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d3eb5baff409f733e759871efd4ee2ec7b776787 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:465c60fb8be6cb25f820c9dcc462df06f973d03ee40f2b5968e1b91d4016af54 +size 1152499 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14f4b601ef9781614417077b78524193b309f517 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df7000ad25a0ae7e07a93d59d2ef5951cde4a989edd3763a5830c026b03493ea +size 1136363 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ba73b581407be07aaac3e7226f31d9bf3cbd8d9 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6581231703b0f374d0369ade51de91bb5ff149d5e9481b5d2e6b4fc2a3e94e +size 1227690 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..874cba567242ae8707c3b9cf0eb67d9a29e3ae6a --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b915a87c852375007f764e85b3f4d657656758a473755f307c82a18f9cf15b3 +size 1357981 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29ca282b6234ab9ebbb0920776b60c2e8603d5fc --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:811955fe3a96368a3ead687220eb14337c378c6e9d92710b9de75b248a2d3166 +size 1181724 diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..144db8390addb4dbf4aeac30a2d45bb415e063c3 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log @@ -0,0 +1,1105 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736309362.788216]: resetting environment +[INFO] [1736309362.793917]: cleared current plan +[INFO] [1736309362.795053]: moving to home +[INFO] [1736309363.797262]: reached home +[INFO] [1736309363.799409]: reset action history +[INFO] [1736309366.508332]: environment reset +Starting episode 0 +[INFO] [1736309366.510613]: resetting environment +[INFO] [1736309366.513325]: cleared current plan +[INFO] [1736309366.514242]: moving to home +[INFO] [1736309367.515800]: reached home +[INFO] [1736309367.517056]: reset action history +[INFO] [1736309370.226472]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736309382.947214]: resetting environment +[INFO] [1736309382.953502]: cleared current plan +[INFO] [1736309382.954578]: moving to home +[INFO] [1736309383.957289]: reached home +[INFO] [1736309383.958685]: reset action history +[INFO] [1736309386.665722]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_0_2025-01-07-23-09-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:32913/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2249664] +process[print_text-2]: started with pid [2249688] +process[rosbag_record-3]: started with pid [2249689] +started bagging! +For topic gripper_width: timestamp difference is 101059037 for nearest: 1736309408872521239 - target: 1736309408771462202 at idx 249 +For topic gripper_width: timestamp difference is 101225191 for nearest: 1736309409072522932 - target: 1736309408971297741 at idx 249 +For topic gripper_width: timestamp difference is 103813215 for nearest: 1736309412272525593 - target: 1736309412168712378 at idx 249 +[INFO] [1736309413.679024]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736309414.418317]: Stored episode 1. +[INFO] [1736309414.419116]: Saving video... +[INFO] [1736309414.608092]: Video saved! +Starting episode 1 +[INFO] [1736309414.628299]: resetting environment +[INFO] [1736309414.633703]: cleared current plan +[INFO] [1736309414.634024]: moving to home +[INFO] [1736309418.236542]: reached home +[INFO] [1736309418.237334]: reset action history +[INFO] [1736309420.944214]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309426.808811]: resetting environment +[INFO] [1736309426.812714]: cleared current plan +[INFO] [1736309426.812879]: moving to home +[INFO] [1736309427.814708]: reached home +[INFO] [1736309427.815107]: reset action history +[INFO] [1736309430.524013]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_1_2025-01-07-23-10-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:42631/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2249800] +process[print_text-5]: started with pid [2249801] +process[rosbag_record-6]: started with pid [2249802] +started bagging! +For topic gripper_width: timestamp difference is 100500216 for nearest: 1736309450072523703 - target: 1736309449972023487 at idx 249 +For topic gripper_width: timestamp difference is 100697748 for nearest: 1736309450272548906 - target: 1736309450171851158 at idx 249 +For topic gripper_width: timestamp difference is 101530184 for nearest: 1736309451272523751 - target: 1736309451170993567 at idx 249 +[INFO] [1736309457.471862]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736309458.255036]: Stored episode 2. +[INFO] [1736309458.255502]: Saving video... +[INFO] [1736309458.699598]: Video saved! +Starting episode 2 +[INFO] [1736309458.735729]: resetting environment +[INFO] [1736309458.759054]: cleared current plan +[INFO] [1736309458.760156]: moving to home +[INFO] [1736309462.365168]: reached home +[INFO] [1736309462.375918]: reset action history +[INFO] [1736309465.147839]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309472.640237]: resetting environment +[INFO] [1736309472.652666]: cleared current plan +[INFO] [1736309472.652943]: moving to home +[INFO] [1736309473.654621]: reached home +[INFO] [1736309473.658164]: reset action history +[INFO] [1736309476.366796]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_2_2025-01-07-23-11-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:39693/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2249935] +process[print_text-8]: started with pid [2249936] +process[rosbag_record-9]: started with pid [2249937] +started bagging! +[INFO] [1736309503.339850]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1736309504.174892]: Stored episode 3. +[INFO] [1736309504.184987]: Saving video... +[INFO] [1736309504.892424]: Video saved! +Starting episode 3 +[INFO] [1736309504.922627]: resetting environment +[INFO] [1736309504.937076]: cleared current plan +[INFO] [1736309504.938453]: moving to home +[WARN] [1736309504.666627]: Plan exhausted +[WARN] [1736309504.709287]: Plan exhausted +[WARN] [1736309504.752079]: Plan exhausted +[WARN] [1736309504.787668]: Plan exhausted +[WARN] [1736309504.826278]: Plan exhausted +[WARN] [1736309504.879425]: Plan exhausted +[WARN] [1736309504.910120]: Plan exhausted +[INFO] [1736309508.547473]: reached home +[INFO] [1736309508.552067]: reset action history +[INFO] [1736309511.329671]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309521.201609]: resetting environment +[INFO] [1736309521.211897]: cleared current plan +[INFO] [1736309521.214448]: moving to home +[INFO] [1736309522.221198]: reached home +[INFO] [1736309522.227138]: reset action history +[INFO] [1736309524.999592]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_3_2025-01-07-23-12-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:33351/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2250100] +process[print_text-11]: started with pid [2250101] +process[rosbag_record-12]: started with pid [2250102] +started bagging! +[INFO] [1736309552.113544]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit[print_text-11] killing on exit +[INFO] [1736309552.922062]: Stored episode 4. +[INFO] [1736309552.925835]: Saving video... +[INFO] [1736309553.626599]: Video saved! +Starting episode 4 +[INFO] [1736309553.677943]: resetting environment +[INFO] [1736309553.699229]: cleared current plan +[INFO] [1736309553.701262]: moving to home +[WARN] [1736309553.466875]: Plan exhausted +[WARN] [1736309553.506736]: Plan exhausted +[WARN] [1736309553.554080]: Plan exhausted +[WARN] [1736309553.590917]: Plan exhausted +[WARN] [1736309553.633152]: Plan exhausted +[WARN] [1736309553.672938]: Plan exhausted +[INFO] [1736309557.009076]: reached home +[INFO] [1736309557.018197]: reset action history +[INFO] [1736309559.819374]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309569.581091]: resetting environment +[INFO] [1736309569.600357]: cleared current plan +[INFO] [1736309569.605320]: moving to home +[INFO] [1736309570.612778]: reached home +[INFO] [1736309570.627564]: reset action history +[INFO] [1736309573.404393]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_4_2025-01-07-23-12-53.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:34719/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2250225] +process[print_text-14]: started with pid [2250226] +process[rosbag_record-15]: started with pid [2250250] +started bagging! +For topic gripper_width: timestamp difference is -116007404 for nearest: 1736309593805856152 - target: 1736309593921863556 at idx 249 +[INFO] [1736309600.377359]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736309601.194555]: Stored episode 5. +[INFO] [1736309601.199376]: Saving video... +[INFO] [1736309601.640723]: Video saved! +Starting episode 5 +[INFO] [1736309601.712692]: resetting environment +[INFO] [1736309601.733608]: cleared current plan +[INFO] [1736309601.738080]: moving to home +[INFO] [1736309605.342427]: reached home +[INFO] [1736309605.350913]: reset action history +[INFO] [1736309608.210175]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309618.420706]: resetting environment +[INFO] [1736309618.478722]: cleared current plan +[INFO] [1736309618.482236]: moving to home +[INFO] [1736309619.492911]: reached home +[INFO] [1736309619.504839]: reset action history +[INFO] [1736309622.302781]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_5_2025-01-07-23-13-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:43691/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2250358] +process[print_text-17]: started with pid [2250359] +process[rosbag_record-18]: started with pid [2250383] +started bagging! +For topic gripper_width: timestamp difference is 114530925 for nearest: 1736309631905855064 - target: 1736309631791324139 at idx 116 +[INFO] [1736309649.325734]: Storing episode... +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[pub_text-16] killing on exit +[INFO] [1736309650.060689]: Stored episode 6. +[INFO] [1736309650.070699]: Saving video... +[INFO] [1736309650.500024]: Video saved! +Starting episode 6 +[INFO] [1736309650.583052]: resetting environment +[INFO] [1736309650.615137]: cleared current plan +[INFO] [1736309650.629166]: moving to home +[INFO] [1736309654.247285]: reached home +[INFO] [1736309654.255549]: reset action history +[INFO] [1736309657.118783]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309662.184369]: resetting environment +[INFO] [1736309662.213783]: cleared current plan +[INFO] [1736309662.216328]: moving to home +[INFO] [1736309663.224345]: reached home +[INFO] [1736309663.230386]: reset action history +[INFO] [1736309666.015098]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_6_2025-01-07-23-14-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:34159/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2250495] +process[print_text-20]: started with pid [2250496] +process[rosbag_record-21]: started with pid [2250520] +started bagging! +For topic gripper_width: timestamp difference is -117843038 for nearest: 1736309674405883425 - target: 1736309674523726463 at idx 115 +For topic gripper_width: timestamp difference is -116641147 for nearest: 1736309675805886405 - target: 1736309675922527552 at idx 134 +[INFO] [1736309693.026764]: Storing episode... +[rosbag_record-21] killing on exit +[pub_text-19] killing on exit +[print_text-20] killing on exit +[INFO] [1736309693.809181]: Stored episode 7. +[INFO] [1736309693.819571]: Saving video... +[INFO] [1736309694.457792]: Video saved! +Starting episode 7 +[INFO] [1736309694.546569]: resetting environment +[INFO] [1736309694.575470]: cleared current plan +[INFO] [1736309694.583441]: moving to home +[WARN] [1736309694.371091]: Plan exhausted +[WARN] [1736309694.412490]: Plan exhausted +[WARN] [1736309694.464384]: Plan exhausted +[WARN] [1736309694.492014]: Plan exhausted +[WARN] [1736309694.533608]: Plan exhausted +[WARN] [1736309694.571106]: Plan exhausted +[INFO] [1736309698.199812]: reached home +[INFO] [1736309698.225264]: reset action history +[INFO] [1736309701.082086]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309707.507246]: resetting environment +[INFO] [1736309707.542182]: cleared current plan +[INFO] [1736309707.545735]: moving to home +[INFO] [1736309708.557367]: reached home +[INFO] [1736309708.564319]: reset action history +[INFO] [1736309711.367043]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_7_2025-01-07-23-15-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:33567/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2250653] +process[print_text-23]: started with pid [2250654] +process[rosbag_record-24]: started with pid [2250655] +started bagging! +[INFO] [1736309738.630509]: Storing episode... +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit +[print_text-23] killing on exit +[INFO] [1736309739.421757]: Stored episode 8. +[INFO] [1736309739.429366]: Saving video... +[INFO] [1736309739.900646]: Video saved! +[WARN] [1736309739.971996]: Plan exhausted +[WARN] [1736309740.012268]: Plan exhausted +Starting episode 8 +[INFO] [1736309739.996704]: resetting environment +[INFO] [1736309740.036652]: cleared current plan +[INFO] [1736309740.042345]: moving to home +[INFO] [1736309743.465983]: reached home +[INFO] [1736309743.473903]: reset action history +[INFO] [1736309746.375606]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309810.397293]: resetting environment +[INFO] [1736309810.434689]: cleared current plan +[INFO] [1736309810.442725]: moving to home +[INFO] [1736309811.464926]: reached home +[INFO] [1736309811.476454]: reset action history +[INFO] [1736309814.320963]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_8_2025-01-07-23-16-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:40549/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2250867] +process[print_text-26]: started with pid [2250868] +process[rosbag_record-27]: started with pid [2250869] +started bagging! +For topic gripper_width: timestamp difference is 102140465 for nearest: 1736309822772519388 - target: 1736309822670378923 at idx 115 +For topic gripper_width: timestamp difference is 102300629 for nearest: 1736309822972518906 - target: 1736309822870218277 at idx 117 +For topic gripper_width: timestamp difference is 105929949 for nearest: 1736309827572519877 - target: 1736309827466589928 at idx 165 +[INFO] [1736309845.387823]: Storing episode... +[rosbag_record-27] killing on exit[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1736309846.191552]: Stored episode 9. +[INFO] [1736309846.207033]: Saving video... +[INFO] [1736309846.513866]: Video saved! +Starting episode 9 +[INFO] [1736309846.606346]: resetting environment +[INFO] [1736309846.615084]: cleared current plan +[INFO] [1736309846.615809]: moving to home +[INFO] [1736309850.030509]: reached home +[INFO] [1736309850.036292]: reset action history +[INFO] [1736309853.003278]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309861.854169]: resetting environment +[INFO] [1736309861.890858]: cleared current plan +[INFO] [1736309861.897283]: moving to home +[INFO] [1736309862.906282]: reached home +[INFO] [1736309862.917070]: reset action history +[INFO] [1736309865.836947]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_9_2025-01-07-23-17-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:43097/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2251003] +process[print_text-29]: started with pid [2251004] +process[rosbag_record-30]: started with pid [2251005] +started bagging! +For topic gripper_width: timestamp difference is -127975759 for nearest: 1736309867805851880 - target: 1736309867933827639 at idx 31 +[INFO] [1736309893.222113]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit +[pub_text-28] killing on exit +[INFO] [1736309894.033147]: Stored episode 10. +[INFO] [1736309894.038860]: Saving video... +[INFO] [1736309894.301781]: Video saved! +Starting episode 10 +[INFO] [1736309894.370502]: resetting environment +[INFO] [1736309894.427521]: cleared current plan +[INFO] [1736309894.435416]: moving to home +[INFO] [1736309897.559939]: reached home +[INFO] [1736309897.577700]: reset action history +[INFO] [1736309900.563271]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309908.023661]: resetting environment +[INFO] [1736309908.081740]: cleared current plan +[INFO] [1736309908.086860]: moving to home +[INFO] [1736309909.099305]: reached home +[INFO] [1736309909.109255]: reset action history +[INFO] [1736309912.069407]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_10_2025-01-07-23-18-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:46849/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2251116] +process[print_text-32]: started with pid [2251140] +process[rosbag_record-33]: started with pid [2251141] +started bagging! +[INFO] [1736309939.097076]: Storing episode... +[rosbag_record-33] killing on exit +[print_text-32] killing on exit +[pub_text-31] killing on exit +[INFO] [1736309939.942763]: Stored episode 11. +[INFO] [1736309939.961628]: Saving video... +[WARN] [1736309940.414467]: Plan exhausted +[WARN] [1736309940.452013]: Plan exhausted +[WARN] [1736309940.492650]: Plan exhausted +[WARN] [1736309940.534355]: Plan exhausted +[INFO] [1736309940.433852]: Video saved! +Starting episode 11 +[INFO] [1736309940.509328]: resetting environment +[INFO] [1736309940.550305]: cleared current plan +[INFO] [1736309940.553222]: moving to home +[INFO] [1736309944.170141]: reached home +[INFO] [1736309944.182426]: reset action history +[INFO] [1736309947.137318]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309952.939165]: resetting environment +[INFO] [1736309952.985415]: cleared current plan +[INFO] [1736309952.995934]: moving to home +[INFO] [1736309954.001412]: reached home +[INFO] [1736309954.005630]: reset action history +[INFO] [1736309956.986075]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_11_2025-01-07-23-19-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:45787/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2251274] +process[print_text-35]: started with pid [2251275] +process[rosbag_record-36]: started with pid [2251276] +started bagging! +[INFO] [1736309989.299902]: Storing episode... +[pub_text-34] killing on exit +[print_text-35] killing on exit +[rosbag_record-36] killing on exit +[INFO] [1736309990.122174]: Stored episode 12. +[INFO] [1736309990.127860]: Saving video... +[INFO] [1736309990.501062]: Video saved! +Starting episode 12 +[INFO] [1736309990.635354]: resetting environment +[INFO] [1736309990.672257]: cleared current plan +[INFO] [1736309990.678658]: moving to home +[WARN] [1736309990.650975]: Plan exhausted +[INFO] [1736309994.308190]: reached home +[INFO] [1736309994.322680]: reset action history +[INFO] [1736309997.396230]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310002.346595]: resetting environment +[INFO] [1736310002.419394]: cleared current plan +[INFO] [1736310002.429585]: moving to home +[INFO] [1736310003.453963]: reached home +[INFO] [1736310003.480619]: reset action history +[INFO] [1736310006.477247]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_12_2025-01-07-23-20-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:34421/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2251409] +process[print_text-38]: started with pid [2251410] +process[rosbag_record-39]: started with pid [2251411] +started bagging! +[INFO] [1736310040.460577]: Storing episode... +[rosbag_record-39] killing on exit +[print_text-38] killing on exit +[pub_text-37] killing on exit +[INFO] [1736310041.253798]: Stored episode 13. +[INFO] [1736310041.297114]: Saving video... +[INFO] [1736310041.800019]: Video saved! +Starting episode 13 +[INFO] [1736310041.920606]: resetting environment +[INFO] [1736310041.957830]: cleared current plan +[INFO] [1736310041.962252]: moving to home +[WARN] [1736310041.815146]: Plan exhausted +[WARN] [1736310041.856696]: Plan exhausted +[WARN] [1736310041.890977]: Plan exhausted +[WARN] [1736310041.932002]: Plan exhausted +[INFO] [1736310045.374320]: reached home +[INFO] [1736310045.382409]: reset action history +[INFO] [1736310048.445317]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310053.085172]: resetting environment +[INFO] [1736310053.140627]: cleared current plan +[INFO] [1736310053.167216]: moving to home +[INFO] [1736310054.189069]: reached home +[INFO] [1736310054.207544]: reset action history +[INFO] [1736310057.210009]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_13_2025-01-07-23-20-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:44103/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2251545] +process[print_text-41]: started with pid [2251546] +process[rosbag_record-42]: started with pid [2251547] +started bagging! +For topic gripper_width: timestamp difference is 100210703 for nearest: 1736310067005852259 - target: 1736310066905641556 at idx 136 +For topic gripper_width: timestamp difference is 104406745 for nearest: 1736310073672478349 - target: 1736310073568071604 at idx 213 +For topic gripper_width: timestamp difference is 104594309 for nearest: 1736310073872505505 - target: 1736310073767911196 at idx 215 +For topic gripper_width: timestamp difference is 106134614 for nearest: 1736310075672517261 - target: 1736310075566382647 at idx 232 +For topic gripper_width: timestamp difference is 108841333 for nearest: 1736310078872517500 - target: 1736310078763676167 at idx 249 +For topic gripper_width: timestamp difference is 109037758 for nearest: 1736310079072547079 - target: 1736310078963509321 at idx 249 +For topic gripper_width: timestamp difference is 109505507 for nearest: 1736310079672498557 - target: 1736310079562993050 at idx 249 +For topic gripper_width: timestamp difference is 109812820 for nearest: 1736310080005848968 - target: 1736310079896036148 at idx 249 +[INFO] [1736310090.217951]: Storing episode... +[print_text-41] killing on exit +[pub_text-40] killing on exit +[rosbag_record-42] killing on exit +[INFO] [1736310091.082097]: Stored episode 14. +[INFO] [1736310091.104192]: Saving video... +[INFO] [1736310091.851470]: Video saved! +Starting episode 14 +[INFO] [1736310091.998303]: resetting environment +[INFO] [1736310092.037174]: cleared current plan +[INFO] [1736310092.039322]: moving to home +[WARN] [1736310091.575853]: Plan exhausted +[WARN] [1736310091.612312]: Plan exhausted +[WARN] [1736310091.666783]: Plan exhausted +[WARN] [1736310091.693486]: Plan exhausted +[WARN] [1736310091.731887]: Plan exhausted +[WARN] [1736310091.772051]: Plan exhausted +[WARN] [1736310091.814666]: Plan exhausted +[WARN] [1736310091.871161]: Plan exhausted +[WARN] [1736310091.913722]: Plan exhausted +[WARN] [1736310091.936446]: Plan exhausted +[WARN] [1736310091.974681]: Plan exhausted +[WARN] [1736310092.011065]: Plan exhausted +[INFO] [1736310095.369313]: reached home +[INFO] [1736310095.397460]: reset action history +[INFO] [1736310098.433733]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310104.251142]: resetting environment +[INFO] [1736310104.310726]: cleared current plan +[INFO] [1736310104.323471]: moving to home +[INFO] [1736310105.332454]: reached home +[INFO] [1736310105.364758]: reset action history +[INFO] [1736310108.450092]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_14_2025-01-07-23-21-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:39191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2251683] +process[print_text-44]: started with pid [2251684] +process[rosbag_record-45]: started with pid [2251685] +started bagging! +[INFO] [1736310141.365065]: Storing episode... +[pub_text-43] killing on exit +[print_text-44] killing on exit +[rosbag_record-45] killing on exit +[INFO] [1736310142.216404]: Stored episode 15. +[INFO] [1736310142.269480]: Saving video... +[WARN] [1736310142.711264]: Plan exhausted +[WARN] [1736310142.731286]: Plan exhausted +[WARN] [1736310142.773924]: Plan exhausted +[INFO] [1736310142.888594]: Video saved! +Starting episode 15 +[INFO] [1736310143.034974]: resetting environment +[INFO] [1736310143.107678]: cleared current plan +[INFO] [1736310143.119665]: moving to home +[WARN] [1736310142.818321]: Plan exhausted +[WARN] [1736310142.851273]: Plan exhausted +[WARN] [1736310142.911096]: Plan exhausted +[WARN] [1736310142.931155]: Plan exhausted +[WARN] [1736310142.971114]: Plan exhausted +[WARN] [1736310143.016486]: Plan exhausted +[WARN] [1736310143.056730]: Plan exhausted +[WARN] [1736310143.097346]: Plan exhausted +[INFO] [1736310146.730998]: reached home +[INFO] [1736310146.741105]: reset action history +[INFO] [1736310149.890322]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310158.082273]: resetting environment +[INFO] [1736310158.178219]: cleared current plan +[INFO] [1736310158.194862]: moving to home +[INFO] [1736310159.207852]: reached home +[INFO] [1736310159.234498]: reset action history +[INFO] [1736310162.325076]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_15_2025-01-07-23-22-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:38569/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2251819] +process[print_text-47]: started with pid [2251820] +process[rosbag_record-48]: started with pid [2251821] +started bagging! +For topic gripper_width: timestamp difference is 110499188 for nearest: 1736310163272514864 - target: 1736310163162015676 at idx 21 +[INFO] [1736310189.642210]: Storing episode... +[rosbag_record-48] killing on exit +[print_text-47] killing on exit +[pub_text-46] killing on exit +[WARN] [1736310190.822164]: Plan exhausted +[INFO] [1736310190.574330]: Stored episode 16. +[INFO] [1736310190.602818]: Saving video... +[INFO] [1736310191.302480]: Video saved! +Starting episode 16 +[INFO] [1736310191.509254]: resetting environment +[INFO] [1736310191.580605]: cleared current plan +[INFO] [1736310191.590549]: moving to home +[WARN] [1736310190.863941]: Plan exhausted +[WARN] [1736310190.891628]: Plan exhausted +[WARN] [1736310190.933470]: Plan exhausted +[WARN] [1736310190.990710]: Plan exhausted +[WARN] [1736310191.025441]: Plan exhausted +[WARN] [1736310191.050978]: Plan exhausted +[WARN] [1736310191.098139]: Plan exhausted +[WARN] [1736310191.131798]: Plan exhausted +[WARN] [1736310191.172624]: Plan exhausted +[WARN] [1736310191.214834]: Plan exhausted +[WARN] [1736310191.259569]: Plan exhausted +[WARN] [1736310191.294756]: Plan exhausted +[WARN] [1736310191.340883]: Plan exhausted +[WARN] [1736310191.379836]: Plan exhausted +[WARN] [1736310191.411004]: Plan exhausted +[WARN] [1736310191.457975]: Plan exhausted +[WARN] [1736310191.495167]: Plan exhausted +[WARN] [1736310191.531399]: Plan exhausted +[WARN] [1736310191.573472]: Plan exhausted +[INFO] [1736310195.222333]: reached home +[INFO] [1736310195.247496]: reset action history +[INFO] [1736310198.455314]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310202.424907]: resetting environment +[INFO] [1736310202.463848]: cleared current plan +[INFO] [1736310202.464073]: moving to home +[INFO] [1736310203.465203]: reached home +[INFO] [1736310203.484339]: reset action history +[INFO] [1736310206.606379]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_16_2025-01-07-23-23-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:42967/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2251954] +process[print_text-50]: started with pid [2251955] +process[rosbag_record-51]: started with pid [2251956] +started bagging! +For topic gripper_width: timestamp difference is 102364842 for nearest: 1736310235972483698 - target: 1736310235870118856 at idx 249 +[INFO] [1736310242.576309]: Storing episode... +[print_text-50] killing on exit +[rosbag_record-51] killing on exit +[pub_text-49] killing on exit +[INFO] [1736310243.427589]: Stored episode 17. +[INFO] [1736310243.448453]: Saving video... +[INFO] [1736310243.834278]: Video saved! +Starting episode 17 +[INFO] [1736310243.991140]: resetting environment +[INFO] [1736310244.061616]: cleared current plan +[INFO] [1736310244.085770]: moving to home +[WARN] [1736310243.931888]: Plan exhausted +[WARN] [1736310243.972859]: Plan exhausted +[WARN] [1736310244.018463]: Plan exhausted +[WARN] [1736310244.050972]: Plan exhausted +[INFO] [1736310247.748381]: reached home +[INFO] [1736310247.772275]: reset action history +[INFO] [1736310250.976659]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310258.407902]: resetting environment +[INFO] [1736310258.453520]: cleared current plan +[INFO] [1736310258.457471]: moving to home +[INFO] [1736310259.475949]: reached home +[INFO] [1736310259.487835]: reset action history +[INFO] [1736310262.668148]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_17_2025-01-07-23-24-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:45603/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2252091] +process[print_text-53]: started with pid [2252092] +process[rosbag_record-54]: started with pid [2252093] +started bagging! +For topic gripper_width: timestamp difference is -160845548 for nearest: 1736310278705840796 - target: 1736310278866686344 at idx 196 +[INFO] [1736310298.567480]: Storing episode... +[rosbag_record-54] killing on exit[pub_text-52] killing on exit +[print_text-53] killing on exit +[INFO] [1736310299.446734]: Stored episode 18. +[INFO] [1736310299.473687]: Saving video... +[INFO] [1736310299.896122]: Video saved! +Starting episode 18 +[INFO] [1736310300.100713]: resetting environment +[INFO] [1736310300.161942]: cleared current plan +[INFO] [1736310300.164238]: moving to home +[WARN] [1736310299.893407]: Plan exhausted +[WARN] [1736310299.931085]: Plan exhausted +[WARN] [1736310299.971323]: Plan exhausted +[WARN] [1736310300.013350]: Plan exhausted +[WARN] [1736310300.066162]: Plan exhausted +[WARN] [1736310300.105778]: Plan exhausted +[WARN] [1736310300.136896]: Plan exhausted +[INFO] [1736310303.581723]: reached home +[INFO] [1736310303.599751]: reset action history +[INFO] [1736310306.794423]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310311.287097]: resetting environment +[INFO] [1736310311.419175]: cleared current plan +[INFO] [1736310311.434439]: moving to home +[INFO] [1736310312.462947]: reached home +[INFO] [1736310312.471092]: reset action history +[INFO] [1736310315.698244]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_18_2025-01-07-23-25-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:33399/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2252226] +process[print_text-56]: started with pid [2252227] +process[rosbag_record-57]: started with pid [2252228] +started bagging! +For topic gripper_width: timestamp difference is 110357616 for nearest: 1736310328405847642 - target: 1736310328295490026 at idx 174 +For topic gripper_width: timestamp difference is 112812677 for nearest: 1736310331605805793 - target: 1736310331492993116 at idx 211 +[INFO] [1736310352.611763]: Storing episode... +[pub_text-55] killing on exit[print_text-56] killing on exit +[rosbag_record-57] killing on exit +[INFO] [1736310353.464476]: Stored episode 19. +[INFO] [1736310353.507863]: Saving video... +[INFO] [1736310354.221335]: Video saved! +Starting episode 19 +[INFO] [1736310354.403825]: resetting environment +[INFO] [1736310354.449260]: cleared current plan +[INFO] [1736310354.452031]: moving to home +[WARN] [1736310353.932970]: Plan exhausted +[WARN] [1736310353.980191]: Plan exhausted +[WARN] [1736310354.025904]: Plan exhausted +[WARN] [1736310354.051225]: Plan exhausted +[WARN] [1736310354.092749]: Plan exhausted +[WARN] [1736310354.139751]: Plan exhausted +[WARN] [1736310354.174092]: Plan exhausted +[WARN] [1736310354.231861]: Plan exhausted +[WARN] [1736310354.259775]: Plan exhausted +[WARN] [1736310354.290956]: Plan exhausted +[WARN] [1736310354.336391]: Plan exhausted +[WARN] [1736310354.377677]: Plan exhausted +[WARN] [1736310354.418255]: Plan exhausted +[INFO] [1736310357.881278]: reached home +[INFO] [1736310357.891325]: reset action history +[INFO] [1736310361.149581]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736310368.043555]: resetting environment +[INFO] [1736310368.149816]: cleared current plan +[INFO] [1736310368.160295]: moving to home +[INFO] [1736310369.181212]: reached home +[INFO] [1736310369.194022]: reset action history +[INFO] [1736310372.338982]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/episode_rosbags/episode_19_2025-01-07-23-26-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2249399.log +started roslaunch server http://158.130.50.37:39645/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2252363] +process[print_text-59]: started with pid [2252364] +process[rosbag_record-60]: started with pid [2252365] +started bagging! +For topic gripper_width: timestamp difference is -101164247 for nearest: 1736310396505869721 - target: 1736310396607033968 at idx 249 +For topic gripper_width: timestamp difference is 102778466 for nearest: 1736310401505847247 - target: 1736310401403068781 at idx 249 +For topic gripper_width: timestamp difference is -230239907 for nearest: 1736310401605798206 - target: 1736310401836038113 at idx 249 +[INFO] [1736310408.462503]: Storing episode... +[rosbag_record-60] killing on exit +[print_text-59] killing on exit[pub_text-58] killing on exit +[INFO] [1736310409.389311]: Stored episode 20. +[INFO] [1736310409.417879]: Saving video... +[INFO] [1736310410.146973]: Video saved! +[INFO] [1736310410.319026]: resetting environment +[INFO] [1736310410.374347]: cleared current plan +[INFO] [1736310410.413050]: moving to home +[WARN] [1736310409.811622]: Plan exhausted +[WARN] [1736310409.851091]: Plan exhausted +[WARN] [1736310409.892719]: Plan exhausted +[WARN] [1736310409.947303]: Plan exhausted +[WARN] [1736310409.988934]: Plan exhausted +[WARN] [1736310410.011648]: Plan exhausted +[WARN] [1736310410.051111]: Plan exhausted +[WARN] [1736310410.091540]: Plan exhausted +[WARN] [1736310410.130985]: Plan exhausted +[WARN] [1736310410.174086]: Plan exhausted +[WARN] [1736310410.217864]: Plan exhausted +[WARN] [1736310410.252315]: Plan exhausted +[WARN] [1736310410.295085]: Plan exhausted +[WARN] [1736310410.335237]: Plan exhausted +[INFO] [1736310413.852170]: reached home +[INFO] [1736310413.883428]: reset action history +[INFO] [1736310417.155069]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8e99f031f0b5102fdc7dc19a906af31f25104f67 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T04:09:15.641385", + "startedAt": "2025-01-08T04:09:15.286608", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.159125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.081, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.971, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.991, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 736.9039688110352 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b603af08801c8117f3de95a2e4c8e2809c0eefd4 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "da5d8645307c977c2178ad77a7431d6a1bfe84d48f087236fd6a919884f4fd31", "size": 1123217, "path": "media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4"}, "global_step": 2659, "_timestamp": 1736310423.3833332, "_runtime": 1068.0891063213348, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "a6cba8accd20e1b8ec9e4ca9adb5f90d53eedbfed622cbc55026038c3eeb3bf4", "size": 1140024, "path": "media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4"}, "eval/num_success": 11.0, "episode": 165.0, "eval/success_rate": 0.550000011920929, "eval/success_rate_lb_0.01": 0.28449782729148865, "eval/success_rate_ub_0.01": 0.7815009355545044, "eval/success_rate_lb_0.025": 0.31899210810661316, "eval/success_rate_ub_0.025": 0.7373047471046448, "eval/success_rate_lb_0.05": 0.3815420866012573, "eval/success_rate_ub_0.05": 0.7210295796394348, "eval/success_rate_lb_0.1": 0.40086737275123596, "eval/success_rate_ub_0.1": 0.704771876335144, "eval/2_eval": {"_type": "video-file", "sha256": "b724e203bcdc67aa5a2f04fd8853c3feb4a28749a40adf615618fd51313c8103", "size": 1138364, "path": "media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "7a1f2288b71555e1168140547170669ba0fd7488f4b222591ceed7f4ba76eb2f", "size": 1148121, "path": "media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "642bdc6d95c526c17f0731df62950ff1c8a72317347fe989e1beacec0f890218", "size": 1136660, "path": "media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "465c60fb8be6cb25f820c9dcc462df06f973d03ee40f2b5968e1b91d4016af54", "size": 1152499, "path": "media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "df7000ad25a0ae7e07a93d59d2ef5951cde4a989edd3763a5830c026b03493ea", "size": 1136363, "path": "media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "9c6581231703b0f374d0369ade51de91bb5ff149d5e9481b5d2e6b4fc2a3e94e", "size": 1227690, "path": "media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "1b915a87c852375007f764e85b3f4d657656758a473755f307c82a18f9cf15b3", "size": 1357981, "path": "media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "811955fe3a96368a3ead687220eb14337c378c6e9d92710b9de75b248a2d3166", "size": 1181724, "path": "media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "b875e5dd12f721236870e25266e34d93e52e8918f9f496a9c963163a64d2489e", "size": 1152134, "path": "media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "c3e8fea09465a60cb8c5b576f7b7880e547ee963d3ed592e68d2993e233d3c1f", "size": 1337182, "path": "media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "84f1b7d03de03c92ec2c8a41dcf82819dd1d23781bc8a7035a2d0186430c5e46", "size": 1441497, "path": "media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "b9f4d6e4d981133c66e545ed42f83f6c0a6bcfd59bcd2dd381b8098c8bdb468f", "size": 1416179, "path": "media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "e423f85e531e4aa2eda45108b3dc3234ecca077473ebbbcf9d7c7a637943ec73", "size": 1366701, "path": "media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "6b8c887cc79fe4bb5aafaaa9f836cc32e9fd296371b4b7b141de76c7f939d76e", "size": 1156265, "path": "media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "01858989d758192b59d620577b9d6cc12c4e50516030cc566a2832125470ad02", "size": 1473520, "path": "media/videos/eval/16_eval_16_01858989d758192b59d6.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "1c3c2ba281fd3f6a37339e1c72ad37f093ad6d75a9aebbf31fdd389f59f0f35b", "size": 1503945, "path": "media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "3d9fd71a9a5e8f25cb3e878d51a4389410db4b2a906bdb1efe163abb52a82fb9", "size": 1579185, "path": "media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "49104c03ad255c9ff4c035a0d9465a2b6ee3ae5ec1832113e329373a9f7e4029", "size": 1494448, "path": "media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "6d6e13da9233b1fe778403493013de6ca8c25beb27ee890a6acc45c2dbb916b0", "size": 401, "artifact_path": "wandb-client-artifact://nulveveqvjuv8hpuotle6g7tu90wktnxw2ijo73t20c8c6qlzn4uk3r2o8fwmgx53hptyyw2149pmd47v0p6ot9p0t46ap5wv0utmfgods4ibwg3b4qi76istljh1a5w/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tuyiuujkd58tdx1m03ze1q5sutrfrq64lpntlpoiip4bt6xqm6og6znn0p65eui65accs8qlzl6q772068rf9ggkw2kegmhif1tzlbd27z6rnsegflhxrse8s1fuwp1y:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 58, "total_num_episodes": 165, "total_success_rate": 0.3515151515151515, "total_success_rate_lb_0.01": 0.2702030539972229, "total_success_rate_ub_0.01": 0.4412672668812916, "total_success_rate_lb_0.025": 0.28281411532931056, "total_success_rate_ub_0.025": 0.42427372933948604, "total_success_rate_lb_0.05": 0.29532181475679786, "total_success_rate_ub_0.05": 0.41212901847331795, "total_success_rate_lb_0.1": 0.30248962160424797, "total_success_rate_ub_0.1": 0.4012609683174675, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "f8b20affcaa8e1150c3bdc06c078340bb44755e14750ad65a01016dd83cccf4f", "size": 526, "artifact_path": "wandb-client-artifact://ciymluzkpl3gbloix57xe63gs98ylymia73h9wkcbxeypdbvzjzjrlsw574tievpgyyu3mtafq26iuns6ekv7qlfpoibik2ukajwsuzmy7fdujm2ag0vbjib3u7ll6ps/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://1hir5g5nfy63ura6m2h3fkbym929uu7nt6jyxwltyk7p5z7taqpopgmh59u5o81hzdvydic6boql09p7kvchnvb9biygdtvu7kdqfj1fhmq8ly173p1ewoa5nwtnm8vr:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1084}, "eval/total_success_rate": 0.35151514410972595, "eval/total_success_rate_lb_0.01": 0.2702030539512634, "eval/total_success_rate_ub_0.01": 0.4412672817707062, "eval/total_success_rate_lb_0.025": 0.2828141152858734, "eval/total_success_rate_ub_0.025": 0.4242737293243408, "eval/total_success_rate_lb_0.05": 0.29532182216644287, "eval/total_success_rate_ub_0.05": 0.4121290147304535, "eval/total_success_rate_lb_0.1": 0.30248960852622986, "eval/total_success_rate_ub_0.1": 0.40126097202301025} \ No newline at end of file diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug-internal.log b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..18f572813b9faa97530846fd5ef897514a3eb249 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug-internal.log @@ -0,0 +1,2274 @@ +2025-01-07 23:09:15,294 INFO StreamThr :2249564 [internal.py:wandb_internal():85] W&B internal server running at pid: 2249564, started at: 2025-01-07 23:09:15.294013 +2025-01-07 23:09:15,295 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status +2025-01-07 23:09:15,307 INFO WriterThread:2249564 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb +2025-01-07 23:09:15,308 DEBUG SenderThread:2249564 [sender.py:send():379] send: header +2025-01-07 23:09:15,316 DEBUG SenderThread:2249564 [sender.py:send():379] send: run +2025-01-07 23:09:15,559 INFO SenderThread:2249564 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files +2025-01-07 23:09:15,559 INFO SenderThread:2249564 [sender.py:_start_run_threads():1188] run started: 1evqtpqc with start time 1736309355.294227 +2025-01-07 23:09:15,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 23:09:15,578 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: check_version +2025-01-07 23:09:15,623 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 23:09:15,628 DEBUG HandlerThread:2249564 [system_info.py:__init__():26] System info init +2025-01-07 23:09:15,628 DEBUG HandlerThread:2249564 [system_info.py:__init__():41] System info init done +2025-01-07 23:09:15,628 INFO HandlerThread:2249564 [system_monitor.py:start():194] Starting system monitor +2025-01-07 23:09:15,628 INFO SystemMonitor:2249564 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 23:09:15,629 INFO HandlerThread:2249564 [system_monitor.py:probe():214] Collecting system info +2025-01-07 23:09:15,629 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 23:09:15,629 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started disk monitoring +2025-01-07 23:09:15,630 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 23:09:15,631 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started memory monitoring +2025-01-07 23:09:15,631 INFO SystemMonitor:2249564 [interfaces.py:start():188] Started network monitoring +2025-01-07 23:09:15,641 DEBUG HandlerThread:2249564 [system_info.py:probe():152] Probing system +2025-01-07 23:09:15,642 DEBUG HandlerThread:2249564 [system_info.py:_probe_git():137] Probing git +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:_probe_git():145] Probing git done +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:probe():200] Probing system done +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T04:09:15.641385', 'startedAt': '2025-01-08T04:09:15.286608', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.159125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.081, 'min': 800.0, 'max': 4700.0}, {'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 736.9039688110352}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 23:09:15,647 INFO HandlerThread:2249564 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 23:09:15,647 INFO HandlerThread:2249564 [system_monitor.py:probe():227] Publishing system info +2025-01-07 23:09:15,647 DEBUG HandlerThread:2249564 [system_info.py:_save_code():44] Saving code +2025-01-07 23:09:15,649 DEBUG HandlerThread:2249564 [system_info.py:_save_code():67] Saving code done +2025-01-07 23:09:15,649 DEBUG HandlerThread:2249564 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 23:09:15,710 DEBUG HandlerThread:2249564 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 23:09:15,710 INFO HandlerThread:2249564 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 23:09:15,713 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 23:09:15,714 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 23:09:15,941 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 23:09:15,941 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: python_packages +2025-01-07 23:09:15,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:15,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:15,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:15,977 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ldm9zc23-wandb-metadata.json +2025-01-07 23:09:15,985 INFO HandlerThread:2249564 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/tb" + save: true +} +control { + relay_id: "0e59b649b6e646d485666f1a9a652098" +} +_info { + stream_id: "1evqtpqc" +} + +2025-01-07 23:09:15,985 DEBUG HandlerThread:2249564 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 23:09:16,013 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rtz14662-diff.patch +2025-01-07 23:09:16,031 DEBUG SenderThread:2249564 [sender.py:send():379] send: telemetry +2025-01-07 23:09:16,035 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/erpa07wq-code/FISH/eval_robot.py +2025-01-07 23:09:16,036 DEBUG SenderThread:2249564 [sender.py:send():379] send: tbrecord +2025-01-07 23:09:16,037 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:09:16,037 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 with policy live +2025-01-07 23:09:16,242 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/chj417hk-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json +2025-01-07 23:09:16,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code +2025-01-07 23:09:16,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH +2025-01-07 23:09:16,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:17,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:18,560 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:18,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:19,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:20,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:20,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:21,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:22,788 DEBUG SenderThread:2249564 [sender.py:send():379] send: config +2025-01-07 23:09:22,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:23,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:24,562 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:24,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:25,943 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:25,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:26,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:26,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:27,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:28,563 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:28,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:29,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:30,564 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:30,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:30,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:30,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:31,015 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:31,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:32,564 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:32,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:33,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:34,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:35,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:36,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:36,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:37,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:38,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:39,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:40,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:41,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:41,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:42,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:43,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:44,567 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:44,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:45,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:09:45,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:09:45,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:46,568 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:46,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:46,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:47,568 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:09:47,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:48,569 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:09:48,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:49,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:50,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:51,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:52,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:52,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:53,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:54,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:55,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:56,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:57,944 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:09:57,949 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:58,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:09:59,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:00,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:00,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:00,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:01,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:02,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:03,021 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:03,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:04,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:05,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:06,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:07,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:08,879 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:08,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:09,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:10,574 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:10,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:11,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:12,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:13,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:14,418 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:14,575 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:14,624 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:10:14,624 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 with policy now +2025-01-07 23:10:14,625 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:10:14,626 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:10:14,626 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:10:14,626 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:10:14,912 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/7hyyubkf-media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:14,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media +2025-01-07 23:10:15,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:10:15,631 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 23:10:15,632 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:10:15,874 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/y7kph9t8-media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:10:15,942 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:15,942 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:15,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:16,576 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:16,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:17,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:18,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:19,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:20,277 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:20,578 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:20,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:21,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:22,578 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:22,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:23,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:24,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:25,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:25,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:26,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:27,580 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:10:27,853 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/uabotbvt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:10:27,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:28,580 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:28,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:29,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:30,581 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:30,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:30,947 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:30,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:30,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:31,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:32,582 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:32,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:33,954 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:34,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:35,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:36,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:36,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:37,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:38,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:39,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:40,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:41,946 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:41,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:42,955 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:43,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:44,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:45,633 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:10:45,945 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:10:45,945 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:10:45,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:46,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:47,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:47,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:48,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:49,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:50,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:51,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:52,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:52,588 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:52,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:53,956 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:54,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:55,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:56,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:57,472 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:10:57,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:58,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:10:58,730 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:10:58,730 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 with policy now +2025-01-07 23:10:58,733 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:10:58,733 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:10:58,733 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:10:58,734 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:10:58,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:10:58,987 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/c1usdr64-media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:10:59,590 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,591 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:10:59,892 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/zladk6ho-media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:10:59,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:00,591 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:00,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:00,950 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:00,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:01,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:02,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:03,421 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:03,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:04,592 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:04,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:05,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:06,593 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:06,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:07,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:08,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:08,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:09,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:10,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:11,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:12,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:13,399 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:11:13,399 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:11:13,400 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:11:13,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:11:13,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:11:13,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:14,595 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:14,697 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:14,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:15,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:11:15,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:15,948 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:15,989 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:16,596 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:16,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:17,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:18,597 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:18,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:19,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:20,631 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:20,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:21,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:22,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:23,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:24,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:25,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:25,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:26,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:27,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:28,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:29,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:30,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:30,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:30,948 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:30,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:31,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:32,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:33,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:34,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:35,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:35,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:36,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:37,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:38,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:39,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:40,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:40,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:41,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:42,967 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:43,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:44,605 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:44,920 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:11:44,920 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 with policy now +2025-01-07 23:11:44,921 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:11:44,921 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:11:44,921 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:11:44,922 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:11:44,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:45,173 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/a0bqb8rd-media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:11:45,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:11:45,899 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/gvum62ui-media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:11:45,950 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:11:45,951 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:11:45,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:46,037 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:46,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:46,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:47,606 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:47,975 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:48,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:49,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:50,607 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:50,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:51,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:51,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:52,608 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:11:52,982 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:53,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:54,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:55,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:56,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:11:56,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:57,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:58,979 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:11:59,982 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:00,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:00,953 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:00,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:01,238 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:01,238 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:01,239 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:01,611 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:01,611 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:01,897 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/382o6bh8-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:01,983 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:02,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:02,612 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:02,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:03,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:04,612 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:04,988 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:05,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:06,613 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:06,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:07,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:07,987 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:08,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:09,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:10,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:11,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:12,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:13,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:13,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:14,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:15,634 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:12:15,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:15,951 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:15,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:16,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:17,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:18,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:19,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:19,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:20,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:21,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:22,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:23,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:24,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:24,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:25,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:26,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:27,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:28,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:29,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:30,951 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:30,952 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:30,993 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:31,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:31,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:32,998 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:33,665 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:12:33,665 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 with policy now +2025-01-07 23:12:33,668 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:12:33,668 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:33,668 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:33,670 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:33,957 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/imyk92rb-media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:33,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:34,622 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:34,622 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:34,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:34,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:12:34,921 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/49axge9l-media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:12:35,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:35,623 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:35,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:36,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:37,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:38,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:38,624 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:38,998 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:39,997 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:40,999 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:41,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:42,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:42,625 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:43,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:44,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:45,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:45,635 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:12:45,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:12:45,964 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:12:46,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:47,003 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:47,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:48,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:49,011 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:49,627 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:12:50,004 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:50,295 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:12:50,296 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:12:50,296 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:12:50,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:12:50,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:51,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:52,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:52,628 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:52,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:53,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:54,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:54,629 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:12:55,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:56,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:57,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:58,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:12:58,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:12:59,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:00,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:00,952 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:00,952 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:01,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:02,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:03,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:04,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:04,102 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:05,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:06,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:07,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:08,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:09,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:09,961 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:10,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:11,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:12,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:13,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:14,041 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:15,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:15,041 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:15,635 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:13:15,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:15,958 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:16,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:16,637 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:17,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:18,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:19,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:20,010 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:20,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:21,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:21,701 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:13:21,701 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 with policy now +2025-01-07 23:13:21,705 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:13:21,705 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:13:21,705 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:13:21,706 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:13:21,953 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/mzs6vtga-media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:22,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:22,639 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:13:22,639 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:22,640 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:22,640 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:13:22,944 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/s6gkztjj-media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:13:23,020 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:24,018 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:25,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:25,347 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:26,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:26,641 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:27,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:28,033 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:29,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:30,024 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:30,642 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:30,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:30,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:31,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:31,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:32,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:33,029 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:34,046 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:35,043 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:36,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:36,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:37,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:38,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:38,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:13:38,895 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/7dbonkd8-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:13:39,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:39,355 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:13:39,356 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:13:39,356 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:13:39,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:13:40,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:40,645 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:41,052 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:41,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:42,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:43,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:44,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:44,647 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:45,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:45,636 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:13:45,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:13:45,957 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:13:46,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:47,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:47,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:48,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:49,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:50,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:51,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:52,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:52,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:53,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:54,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:54,651 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:13:55,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:56,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:57,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:57,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:13:58,067 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:13:59,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:00,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:00,957 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:00,957 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:01,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:02,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:03,031 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:03,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:04,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:05,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:06,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:07,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:08,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:08,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:09,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:10,075 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:10,569 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:14:10,570 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 with policy now +2025-01-07 23:14:10,581 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:14:10,581 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:10,582 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:10,582 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:10,656 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:10,657 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:14:10,847 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/t0l6xdru-media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:11,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:11,123 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/iffolz4h-media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:14:12,086 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:12,657 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:13,087 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:13,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:14,092 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:15,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:15,636 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:14:15,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:15,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:16,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:16,658 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:17,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:18,101 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:18,659 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:19,117 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:19,169 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:20,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:21,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:22,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:22,410 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:22,410 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:22,410 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:22,660 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:22,660 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:14:23,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:24,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:24,269 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:24,661 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:25,117 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:26,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:27,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:28,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:28,663 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:29,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:29,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:30,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:30,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:30,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:31,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:32,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:33,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:34,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:34,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:35,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:36,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:36,665 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:37,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:38,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:38,666 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:39,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:40,074 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:40,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:41,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:42,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:43,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:44,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:45,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:45,637 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:14:45,638 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:45,958 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:14:45,958 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:14:46,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:47,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:48,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:49,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:50,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:50,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:51,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:52,111 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:53,116 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:54,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:54,531 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:14:54,531 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 with policy now +2025-01-07 23:14:54,535 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:14:54,535 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:14:54,535 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:14:54,536 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:54,672 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:14:54,785 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/mvotjo30-media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:55,045 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/bnkkep60-media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:14:55,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:55,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:14:56,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:56,673 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:57,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:57,673 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:14:58,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:14:59,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:00,138 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:00,674 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:00,962 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:00,962 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:01,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:01,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:02,137 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:02,675 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:03,135 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:04,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:05,148 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:06,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:06,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:07,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:07,463 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:15:07,464 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:15:07,464 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:15:07,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:15:07,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:15:07,930 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ki06q0ks-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:15:08,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:08,677 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:09,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:10,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:10,678 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:11,169 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:12,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:12,678 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:12,895 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:13,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:14,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:15,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:15,637 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:15:15,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:15,960 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:16,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:17,154 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:18,091 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:18,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:19,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:20,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:21,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:22,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:23,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:23,984 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:24,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:25,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:26,155 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:27,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:28,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:29,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:29,984 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:30,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:30,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:30,961 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:31,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:32,156 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:33,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:34,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:35,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:35,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:36,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:37,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:38,157 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:39,168 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:39,989 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:15:39,989 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 with policy now +2025-01-07 23:15:39,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:15:39,992 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:15:39,992 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:15:39,992 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:15:40,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:40,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:40,247 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/vowh0va8-media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:40,688 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:40,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:15:40,974 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/m6a9zk1j-media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:15:41,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:41,689 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:42,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:42,690 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:43,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:44,184 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:44,690 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:45,187 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:45,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:45,638 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:15:45,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:15:45,980 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:15:46,189 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:47,182 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:48,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:48,692 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:15:49,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:50,202 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:50,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:51,198 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:52,190 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:53,194 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:54,210 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:55,214 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:55,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:15:56,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:57,216 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:58,219 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:15:59,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:00,217 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:00,977 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:00,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:01,064 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:01,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:02,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:03,224 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:04,221 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:05,223 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:06,223 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:07,001 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:07,226 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:08,230 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:09,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:10,232 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:11,236 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:12,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:12,233 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:13,238 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:14,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:15,244 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:15,638 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:16:15,970 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:15,970 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:16,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:17,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:17,243 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:18,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:19,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:20,270 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:21,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:22,255 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:23,007 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:23,264 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:24,258 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:25,265 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:26,268 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:27,263 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:28,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:28,259 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:29,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:30,279 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:30,966 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:30,966 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:31,276 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:32,281 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:33,077 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:33,274 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:34,275 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:35,280 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:36,283 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:37,283 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:38,284 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:39,022 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:39,280 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:40,281 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:41,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:42,302 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:43,300 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:44,025 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:44,311 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:45,303 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:45,639 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:16:45,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:16:45,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:16:46,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:47,298 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:48,317 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:49,083 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:49,326 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:50,349 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:50,592 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:16:50,592 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:16:50,593 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:16:50,714 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:16:50,714 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:16:51,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:52,310 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:52,715 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:16:53,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:54,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:54,327 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:16:55,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:56,324 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:56,716 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:16:57,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:58,328 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:59,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:16:59,658 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:00,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:00,718 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:00,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:00,963 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:01,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:02,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:03,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:04,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:04,719 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:04,991 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:05,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:06,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:07,329 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:08,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:08,721 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:09,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:10,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:10,641 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:11,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:12,330 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:13,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:14,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:15,331 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:15,639 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:17:15,963 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:15,964 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:16,046 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:16,337 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:17,334 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:18,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:19,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:20,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:21,335 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:22,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:22,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:23,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:24,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:25,336 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:26,593 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:17:26,593 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 with policy now +2025-01-07 23:17:26,600 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:17:26,601 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:17:26,602 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:17:26,602 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:17:26,731 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:26,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:17:26,856 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/g2209p0a-media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:27,131 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/thr302eg-media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:17:27,360 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:27,616 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:28,368 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:28,732 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:29,370 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:30,377 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:30,733 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:30,971 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:30,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:31,363 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:32,371 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:33,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:33,363 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:34,380 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:34,736 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:35,368 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:36,390 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:37,372 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:38,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:38,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:39,390 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:40,373 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:41,391 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:42,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:42,652 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:17:42,652 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:17:42,653 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:42,739 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:17:43,002 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/d1jpx36j-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:17:43,387 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:43,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:44,388 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:44,740 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:45,388 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:45,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:17:45,969 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:17:45,969 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:17:46,395 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:46,741 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:47,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:48,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:48,742 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:17:49,073 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:49,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:50,381 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:51,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:52,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:53,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:54,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:55,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:17:55,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:56,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:57,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:58,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:17:59,382 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:00,047 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:00,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:00,965 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:00,966 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:01,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:02,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:03,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:04,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:05,094 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:05,383 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:06,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:07,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:08,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:09,384 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:10,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:11,045 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:11,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:12,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:13,385 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:14,351 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:18:14,351 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 with policy now +2025-01-07 23:18:14,360 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:18:14,360 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:18:14,360 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:18:14,361 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:18:14,400 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:14,608 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/twgpqrl9-media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:14,752 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:18:15,028 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rzsrf72v-media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:18:15,404 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:15,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:18:15,972 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:15,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:16,110 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:16,399 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:16,753 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:17,399 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:18,426 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:18,753 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:19,423 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:20,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:21,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:22,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:22,424 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:22,755 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:23,427 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:24,425 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:25,428 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:26,440 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:27,052 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:27,428 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:28,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:28,706 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:18:28,707 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:18:28,707 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:18:28,757 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:18:28,757 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:28,758 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:18:29,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:30,478 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:30,758 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:30,978 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:30,978 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:31,444 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:32,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:32,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:32,759 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:33,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:34,430 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:34,760 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:18:35,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:36,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:37,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:37,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:38,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:39,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:40,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:41,431 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:42,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:43,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:43,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:44,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:45,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:45,640 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:18:45,972 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:18:45,972 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:18:46,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:47,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:48,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:48,432 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:49,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:50,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:51,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:52,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:53,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:54,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:54,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:55,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:56,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:57,433 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:58,434 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:18:59,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:18:59,444 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:00,436 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:00,505 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:19:00,505 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 with policy now +2025-01-07 23:19:00,505 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:19:00,506 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:00,506 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:00,506 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:00,767 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/s89wjrvk-media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:00,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:19:00,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:00,980 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:01,038 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/yb54cmhj-media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:19:01,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:01,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:02,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:02,770 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:03,451 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:04,129 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:04,452 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:04,771 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:05,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:06,468 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:06,772 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:07,461 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:08,469 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:08,773 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:09,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:09,486 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:10,495 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:11,488 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:12,473 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:13,492 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:13,760 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:13,761 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:13,762 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:13,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:13,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:19:14,025 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/r81g15bt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:19:14,500 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:14,775 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:15,049 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:15,492 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:15,641 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:19:15,983 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:15,983 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:16,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:17,500 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:18,507 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:18,777 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:19,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:20,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:20,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:21,537 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:22,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:22,779 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:23,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:24,517 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:24,779 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:25,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:25,583 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:26,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:27,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:28,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:29,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:30,518 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:30,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:30,974 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:31,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:31,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:32,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:33,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:34,519 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:35,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:36,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:37,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:37,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:38,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:39,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:40,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:41,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:42,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:42,520 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:43,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:44,523 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:45,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:45,641 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:19:45,973 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:19:45,974 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:19:46,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:47,521 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:48,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:48,522 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:49,524 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:50,532 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:50,621 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:19:50,621 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 with policy now +2025-01-07 23:19:50,661 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:19:50,661 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:19:50,662 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:19:50,662 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:19:50,789 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:19:50,789 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:50,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:50,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:19:50,872 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/bphbgl47-media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:51,143 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/eeo2xvy0-media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:19:51,530 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:52,541 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:52,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:53,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:53,552 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:53,790 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:54,551 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:55,556 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:56,543 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:56,792 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:57,542 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:58,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:19:58,557 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:19:58,792 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:19:59,571 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:00,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:01,000 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:01,000 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:01,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:02,562 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:02,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:02,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:20:02,818 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:02,818 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:02,818 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:03,497 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:03,555 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:03,794 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:04,573 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:04,795 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:05,564 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:06,574 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:07,569 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:08,534 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:08,562 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:08,796 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:09,570 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:10,582 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:11,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:12,582 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:13,586 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:13,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:14,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:14,799 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:15,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:15,642 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:20:15,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:15,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:16,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:17,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:18,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:19,105 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:19,577 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:20,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:21,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:22,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:23,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:24,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:25,082 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:25,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:26,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:27,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:28,578 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:29,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:30,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:30,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:30,977 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:31,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:31,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:32,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:33,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:34,579 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:35,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:36,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:37,083 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:37,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:38,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:39,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:40,580 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:41,611 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:41,869 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:20:41,870 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 with policy now +2025-01-07 23:20:41,878 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:20:41,879 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:41,879 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:41,879 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:42,130 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/17k0r1ko-media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:42,606 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:42,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:42,811 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:20:42,964 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:43,146 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/h61tw7jt-media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:20:43,612 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:43,810 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:44,602 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:45,604 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:45,642 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:20:46,008 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:20:46,008 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:20:46,607 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:46,812 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:47,609 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:48,099 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:48,605 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:49,627 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:50,643 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:50,813 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:51,642 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:52,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:53,143 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:53,631 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:53,814 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:20:53,881 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:20:53,881 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:20:53,881 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:20:54,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:54,815 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:20:54,815 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:55,632 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:56,665 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:57,636 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:58,274 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:20:58,638 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:20:58,817 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:20:59,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:00,648 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:01,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:01,013 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:01,671 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:02,667 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:03,411 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:03,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:04,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:04,819 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:05,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:06,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:07,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:08,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:08,821 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:09,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:09,649 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:10,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:11,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:12,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:13,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:14,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:14,824 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:14,877 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:15,643 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:21:15,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:15,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:15,990 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:16,650 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:16,825 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:17,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:18,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:19,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:20,104 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:20,651 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:20,827 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:21,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:22,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:23,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:24,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:25,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:26,100 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:26,652 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:27,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:28,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:29,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:30,668 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:30,836 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:31,025 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:31,025 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:31,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:31,669 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:31,969 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:21:31,969 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 with policy now +2025-01-07 23:21:31,985 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:21:31,985 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:21:31,986 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:21:31,986 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:21:32,226 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/3b41yzzt-media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:32,704 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:32,837 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:32,838 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:21:33,133 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/w7hak22a-media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:21:33,677 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:33,838 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:34,677 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:35,698 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:36,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:36,686 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:36,839 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:37,694 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:38,682 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:39,734 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:40,699 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:40,840 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:41,141 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:41,702 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:42,718 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:43,731 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:44,732 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:44,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:44,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:21:44,943 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:21:44,943 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:21:44,943 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:21:45,104 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/03z4z2vt-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:21:45,643 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:21:45,740 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:45,842 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:21:46,044 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:21:46,044 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:21:46,206 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:46,732 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:46,843 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:47,746 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:48,728 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:49,753 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:50,746 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:50,844 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:51,738 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:52,115 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:52,760 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:53,748 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:54,748 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:54,846 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:55,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:56,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:56,848 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:21:57,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:58,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:21:58,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:21:59,749 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:00,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:01,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:01,002 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:01,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:02,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:03,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:03,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:04,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:05,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:06,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:07,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:08,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:08,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:09,750 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:10,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:11,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:12,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:13,107 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:13,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:14,751 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:15,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:22:15,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:16,002 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:16,002 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:16,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:17,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:18,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:18,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:19,752 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:20,753 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:21,770 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:22,764 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:22,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:23,015 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:22:23,015 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 with policy now +2025-01-07 23:22:23,033 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:22:23,034 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:22:23,034 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:22:23,034 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:22:23,283 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/do2pu68c-media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:23,757 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:23,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:22:24,120 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:24,159 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ny7cwklp-media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:22:24,798 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:24,859 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:25,772 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:25,860 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:26,777 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:27,766 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:28,792 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:28,861 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:29,777 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:29,786 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:30,781 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:30,862 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:31,020 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:31,020 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:31,789 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:32,808 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:33,797 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:34,791 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:35,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:35,809 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:36,807 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:37,810 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:38,005 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:22:38,005 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:22:38,006 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:22:38,820 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:38,866 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:22:39,817 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:40,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:40,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:40,867 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:41,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:42,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:42,868 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:43,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:44,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:44,868 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:22:45,302 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:45,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:22:45,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:46,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:22:46,006 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:22:46,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:47,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:48,814 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:49,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:50,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:51,112 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:51,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:52,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:53,815 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:54,817 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:55,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:56,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:57,112 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:22:57,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:58,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:22:59,824 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:00,833 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:01,006 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:01,006 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:01,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:02,136 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:02,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:03,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:04,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:05,829 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:06,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:07,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:08,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:08,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:09,830 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:10,874 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:10,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:11,463 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:23:11,463 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 with policy now +2025-01-07 23:23:11,490 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:23:11,490 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:23:11,491 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:23:11,491 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:23:11,712 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/dd30u4x4-media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:11,851 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:11,880 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:11,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:23:12,157 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/dgu28mn5-media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:23:12,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:12,887 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:13,595 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:13,881 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:13,885 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:14,865 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:15,644 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:23:15,888 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:16,026 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:16,026 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:16,882 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:16,900 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:17,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:18,874 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:18,883 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:19,501 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:19,891 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:20,884 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:20,891 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:21,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:22,878 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:22,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:22,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:23:23,063 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:23:23,063 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:23:23,064 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:23:23,885 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:23:23,905 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:24,513 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:24,886 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:24,896 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:25,900 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:26,898 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:27,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:28,887 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:28,906 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:29,653 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:29,913 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:30,908 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:31,056 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:31,057 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:31,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:32,932 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:33,941 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:34,934 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:35,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:35,916 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:36,891 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:36,916 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:37,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:38,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:39,922 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:40,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:41,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:41,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:42,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:43,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:44,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:45,645 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:23:45,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:46,013 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:23:46,013 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:23:46,166 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:46,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:47,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:48,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:49,923 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:50,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:51,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:52,131 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:52,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:53,924 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:54,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:55,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:56,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:57,132 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:23:57,899 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:23:57,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:58,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:23:59,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:00,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:01,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:01,015 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:01,926 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:02,133 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:02,947 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:03,964 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:24:03,964 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 with policy now +2025-01-07 23:24:03,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:04,021 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:24:04,022 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:24:04,022 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:24:04,022 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:24:04,233 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/u2zwnre1-media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:04,911 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:24:04,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:04,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:24:04,953 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:05,223 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/51cglfgr-media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:24:05,912 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:05,976 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:06,948 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:07,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:07,960 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:08,980 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:09,913 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:09,995 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:10,986 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:11,999 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:12,992 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:13,142 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:13,915 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:14,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:15,019 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:15,646 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:24:15,990 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:16,065 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:16,065 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:17,011 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:18,018 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:18,222 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:18,917 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:24:19,016 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:19,132 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:24:19,132 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:24:19,132 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:24:19,189 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/ye62zc23-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:24:19,918 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:24:19,918 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:20,023 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:21,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:21,919 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:22,005 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:23,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:23,785 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:24,014 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:25,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:25,920 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:26,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:27,051 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:28,063 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:29,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:29,164 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:30,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:31,057 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:31,057 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:31,105 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:32,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:33,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:33,924 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:34,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:34,835 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:35,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:36,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:37,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:38,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:39,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:40,009 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:40,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:41,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:41,928 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:24:42,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:43,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:44,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:45,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:45,165 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:45,646 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:24:46,031 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:24:46,031 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:24:46,077 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:47,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:48,035 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:49,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:50,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:50,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:51,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:52,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:53,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:54,036 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:55,037 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:56,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:56,162 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:24:57,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:58,038 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:59,053 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:24:59,942 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:00,038 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:25:00,038 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 with policy now +2025-01-07 23:25:00,042 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:25:00,043 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:00,043 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:00,044 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:00,078 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:00,307 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/1s2u1h5y-media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:00,942 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:00,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:00,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:25:01,051 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:01,051 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:01,070 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:01,162 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:01,239 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/io1ylyl0-media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:25:01,943 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:02,067 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:03,069 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:04,084 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:05,080 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:05,944 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:06,096 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:06,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:07,068 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:08,073 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:09,085 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:09,946 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:10,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:11,113 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:11,176 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:11,194 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:11,194 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:11,194 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:11,947 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:11,947 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:25:12,128 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:13,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:13,948 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:14,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:15,121 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:15,647 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:25:16,044 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:16,044 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:16,098 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:16,180 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:17,106 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:17,950 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:18,095 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:19,119 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:20,102 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:21,119 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:22,144 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:22,175 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:23,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:24,132 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:25,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:26,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:27,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:27,954 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:28,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:28,172 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:29,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:29,955 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:30,122 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:31,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:31,034 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:31,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:32,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:33,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:33,669 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:33,957 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:34,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:35,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:36,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:37,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:38,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:39,123 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:39,176 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:40,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:41,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:42,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:43,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:44,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:45,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:45,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:45,647 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:25:46,034 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:25:46,035 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:25:46,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:47,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:48,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:49,124 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:50,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:50,174 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:51,125 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:52,126 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:53,139 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:53,966 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:54,140 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:54,341 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:25:54,341 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 with policy now +2025-01-07 23:25:54,365 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:25:54,366 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:25:54,366 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:25:54,366 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:25:54,606 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/d4r27cpb-media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:54,967 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:25:55,148 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:55,287 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/m7ke0vqm-media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:25:55,452 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:25:55,968 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:25:56,151 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:57,175 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:58,173 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:59,163 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:25:59,969 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:00,185 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:00,925 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:01,078 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:01,078 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:01,177 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:02,187 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:03,186 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:03,971 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:04,205 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:05,189 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:06,207 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:06,218 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:07,205 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:07,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:26:08,212 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:08,266 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:26:08,266 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:26:08,266 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:26:08,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:26:08,973 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:26:09,229 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:09,974 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:10,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:11,221 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:11,241 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:11,974 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:12,227 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:13,243 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:13,975 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:14,272 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:15,248 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:15,648 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:26:16,075 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:16,075 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:16,231 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:16,266 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:17,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:18,271 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:19,265 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:20,249 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:21,239 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:21,285 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:22,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:23,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:23,980 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:24,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:25,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:26,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:27,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:27,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:28,250 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:29,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:30,258 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:31,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:31,050 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:31,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:32,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:32,251 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:33,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:34,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:35,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:36,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:37,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:37,733 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:37,986 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:38,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:39,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:40,254 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:41,252 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:42,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:42,930 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:43,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:43,989 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:44,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:45,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:45,648 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:26:46,050 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:26:46,050 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:26:46,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:47,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:48,191 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:48,253 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:49,310 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:49,991 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:50,276 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:50,296 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:26:50,296 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 with policy now +2025-01-07 23:26:50,341 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:26:50,341 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:26:50,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:26:50,342 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:26:50,566 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/da34jd6f-media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:50,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval +2025-01-07 23:26:51,277 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:51,293 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/tp4pa9fo-media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:26:51,992 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:52,284 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:53,217 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:53,278 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:54,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:55,282 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:55,994 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:26:56,351 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:57,297 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:58,298 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:59,215 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:26:59,338 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:26:59,996 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:00,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:01,066 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:27:01,066 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:27:01,328 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:01,329 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:01,329 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:01,330 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:01,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:01,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:02,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:02,629 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:27:02,629 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:27:02,968 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:27:02,997 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval +2025-01-07 23:27:02,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table +2025-01-07 23:27:03,224 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/8ue5b6ml-media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:03,312 INFO wandb-upload_0:2249564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6msy2ivh +2025-01-07 23:27:03,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:03,476 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:27:03,486 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:27:03,711 INFO SenderThread:2249564 [sender.py:send_request_log_artifact():1525] logged artifact run-1evqtpqc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjY0MjUxMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkyNzQzOA==', 'latestArtifact': None}} +2025-01-07 23:27:03,711 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:27:03,711 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json with policy now +2025-01-07 23:27:03,712 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:03,712 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,713 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send():379] send: telemetry +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,713 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,713 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,715 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,715 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,716 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,716 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,716 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,717 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,717 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,717 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,717 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,718 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,718 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,718 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,719 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,719 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,719 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,719 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,720 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,720 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:03,720 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:03,720 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:27:03,998 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:03,999 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:04,000 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval +2025-01-07 23:27:04,002 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/x9oxhdwl-media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:04,234 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/at5tb6y0-media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:04,282 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/vhynu0g9-events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:04,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:04,518 INFO wandb-upload_0:2249564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1xjl6479 +2025-01-07 23:27:04,898 INFO SenderThread:2249564 [sender.py:send_request_log_artifact():1525] logged artifact run-1evqtpqc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjY0MjUyMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkyNzQ0MQ==', 'latestArtifact': None}} +2025-01-07 23:27:04,901 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:05,013 DEBUG SenderThread:2249564 [sender.py:send():379] send: files +2025-01-07 23:27:05,014 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json with policy now +2025-01-07 23:27:05,014 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:05,014 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:05,014 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:05,291 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /tmp/tmpwboe2p40wandb/rg32mlkp-media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:05,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:06,017 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:06,017 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:06,018 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:27:06,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:07,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:08,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:09,319 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:10,220 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:10,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:11,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:12,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:13,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:14,320 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:15,220 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:15,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:15,649 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:27:16,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:27:16,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:27:16,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:17,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:18,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:19,321 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:20,053 DEBUG SenderThread:2249564 [sender.py:send():379] send: exit +2025-01-07 23:27:20,053 INFO SenderThread:2249564 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_exit():588] handling runtime: 1084 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_exit():594] send defer +2025-01-07 23:27:20,054 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,054 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 23:27:20,054 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 23:27:20,054 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 1 +2025-01-07 23:27:20,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 23:27:20,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,055 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 23:27:20,055 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 2 +2025-01-07 23:27:20,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 23:27:20,055 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 23:27:20,055 DEBUG SystemMonitor:2249564 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 23:27:20,055 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 23:27:20,056 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 23:27:20,107 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 23:27:20,108 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 23:27:20,108 INFO HandlerThread:2249564 [interfaces.py:finish():200] Joined network monitor +2025-01-07 23:27:20,108 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,108 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 23:27:20,108 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 3 +2025-01-07 23:27:20,108 DEBUG SenderThread:2249564 [sender.py:send():379] send: stats +2025-01-07 23:27:20,108 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,109 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 23:27:20,109 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:20,109 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 23:27:20,109 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 4 +2025-01-07 23:27:20,109 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:20,109 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 23:27:21,025 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:22,025 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,340 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 23:27:26,341 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,341 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,342 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 23:27:26,342 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 5 +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send():379] send: history +2025-01-07 23:27:26,342 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,342 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 23:27:26,342 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:27:26,343 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:26,343 DEBUG SenderThread:2249564 [sender.py:send():379] send: summary +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:27:26,344 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 6 +2025-01-07 23:27:26,344 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,344 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 23:27:26,344 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,344 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 23:27:26,345 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 7 +2025-01-07 23:27:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:27:26,345 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:26,345 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 23:27:26,345 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:26,345 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 23:27:27,028 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:27,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:27,613 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 8 +2025-01-07 23:27:27,613 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:27,614 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:27,614 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 23:27:27,614 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:27,614 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 23:27:27,614 INFO SenderThread:2249564 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 23:27:27,616 INFO SenderThread:2249564 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 23:27:27,618 INFO SenderThread:2249564 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 23:27:27,629 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 9 +2025-01-07 23:27:27,630 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:27,630 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 23:27:27,630 DEBUG SenderThread:2249564 [sender.py:send():379] send: artifact +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 23:27:27,885 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 23:27:27,885 INFO SenderThread:2249564 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 23:27:28,028 INFO Thread-12 :2249564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml config.yaml +2025-01-07 23:27:28,029 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json wandb-summary.json +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt requirements.txt +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-metadata.json wandb-metadata.json +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 events.out.tfevents.1736309355.leonmkim-ROG-Strix-G15CS-G15CS.2249399.0 +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log output.log +2025-01-07 23:27:28,030 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/diff.patch diff.patch +2025-01-07 23:27:28,031 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 media/videos/eval/16_eval_16_01858989d758192b59d6.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 media/videos/eval/2_eval_2_b724e203bcdc67aa5a2f.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 media/videos/eval/3_eval_3_7a1f2288b71555e11681.mp4 +2025-01-07 23:27:28,037 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 media/videos/eval/1_eval_1_a6cba8accd20e1b8ec9e.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 media/videos/eval/14_eval_14_e423f85e531e4aa2eda4.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 media/videos/eval/0_eval_0_da5d8645307c977c2178.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 media/videos/eval/4_eval_4_642bdc6d95c526c17f07.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 media/videos/eval/12_eval_12_84f1b7d03de03c92ec2c.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 media/videos/eval/9_eval_9_811955fe3a96368a3ead.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 media/videos/eval/15_eval_15_6b8c887cc79fe4bb5aaf.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 media/videos/eval/8_eval_8_1b915a87c852375007f7.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 media/videos/eval/7_eval_7_9c6581231703b0f374d0.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 media/videos/eval/13_eval_13_b9f4d6e4d981133c66e5.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 media/videos/eval/11_eval_11_c3e8fea09465a60cb8c5.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 media/videos/eval/5_eval_5_465c60fb8be6cb25f820.mp4 +2025-01-07 23:27:28,038 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 media/videos/eval/17_eval_17_1c3c2ba281fd3f6a3733.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 media/videos/eval/18_eval_18_3d9fd71a9a5e8f25cb3e.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 media/videos/eval/19_eval_19_49104c03ad255c9ff4c0.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 media/videos/eval/10_eval_10_b875e5dd12f721236870.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 media/videos/eval/6_eval_6_df7000ad25a0ae7e07a9.mp4 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json media/table/eval/success_rate_ci_20_6d6e13da9233b1fe7784.table.json +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json media/table/eval/total_success_rate_ci_21_f8b20affcaa8e1150c3b.table.json +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 10 +2025-01-07 23:27:28,039 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,039 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 23:27:28,039 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 23:27:28,039 INFO SenderThread:2249564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:27:28,055 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,055 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,214 INFO wandb-upload_3:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/config.yaml +2025-01-07 23:27:28,227 INFO wandb-upload_0:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/wandb-summary.json +2025-01-07 23:27:28,243 INFO wandb-upload_2:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/output.log +2025-01-07 23:27:28,252 INFO wandb-upload_1:2249564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/files/requirements.txt +2025-01-07 23:27:28,453 INFO Thread-11 (_thread_body):2249564 [sender.py:transition_state():614] send defer: 11 +2025-01-07 23:27:28,453 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,453 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 23:27:28,453 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,453 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 23:27:28,453 INFO SenderThread:2249564 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 23:27:28,456 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 12 +2025-01-07 23:27:28,457 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,457 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 23:27:28,457 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,457 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 23:27:28,457 INFO SenderThread:2249564 [file_stream.py:finish():601] file stream finish called +2025-01-07 23:27:28,655 INFO SenderThread:2249564 [file_stream.py:finish():605] file stream finish is done +2025-01-07 23:27:28,655 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 13 +2025-01-07 23:27:28,655 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,655 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:transition_state():614] send defer: 14 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send():379] send: final +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send():379] send: footer +2025-01-07 23:27:28,656 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:27:28,656 INFO HandlerThread:2249564 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 23:27:28,656 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: defer +2025-01-07 23:27:28,656 INFO SenderThread:2249564 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,657 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 23:27:28,657 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:27:28,658 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: server_info +2025-01-07 23:27:28,658 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 23:27:28,660 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 23:27:28,661 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:27:28,708 DEBUG SenderThread:2249564 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:27:28,708 INFO MainThread:2249564 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 23:27:28,709 INFO MainThread:2249564 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 23:27:28,709 INFO MainThread:2249564 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 23:27:28,709 DEBUG HandlerThread:2249564 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 23:27:28,709 INFO HandlerThread:2249564 [handler.py:finish():882] shutting down handler +2025-01-07 23:27:29,658 INFO WriterThread:2249564 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [sender.py:finish():1615] shutting down sender +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:27:29,708 INFO SenderThread:2249564 [file_pusher.py:join():175] waiting for file pusher diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug.log b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..35e82fc403ed099ce68e5c2e109f7b2d2507964f --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Configure stats pid to 2249399 +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/settings +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 23:09:15,288 INFO MainThread:2249399 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug.log +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/logs/debug-internal.log +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():569] calling init triggers +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc'} +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():619] starting backend +2025-01-07 23:09:15,289 INFO MainThread:2249399 [wandb_init.py:init():623] setting up manager +2025-01-07 23:09:15,293 INFO MainThread:2249399 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 23:09:15,293 INFO MainThread:2249399 [wandb_init.py:init():631] backend started and connected +2025-01-07 23:09:15,307 INFO MainThread:2249399 [wandb_init.py:init():720] updated telemetry +2025-01-07 23:09:15,313 INFO MainThread:2249399 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 23:09:15,578 INFO MainThread:2249399 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 23:09:15,618 INFO MainThread:2249399 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 23:09:15,618 INFO MainThread:2249399 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 23:09:15,943 INFO MainThread:2249399 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 23:09:15,944 INFO MainThread:2249399 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 23:09:15,945 INFO MainThread:2249399 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 23:09:15,945 INFO MainThread:2249399 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/1evqtpqc/tb, True +2025-01-07 23:09:22,787 INFO MainThread:2249399 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodin'} +2025-01-07 23:27:29,710 WARNING MsgRouterThr:2249399 [router.py:message_loop():77] message_loop has been closed diff --git a/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..018dc457c1bcfd65a8e724f1bd248f87c3bb2ae7 --- /dev/null +++ b/1evqtpqc/wandb/run-20250107_230915-1evqtpqc/run-1evqtpqc.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd744c02ee5817d2564bba2e447b98993bd98541dda2bb8ba9be227b9e5e2d14 +size 242417 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52cae6ed27b38b9df86d3b1f7c6086ea16d6b4ed --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1002_0' +wandb_notes: 1002_0_req_1365_0 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/gyo63dh5/.hydra/config.yaml b/gyo63dh5/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/gyo63dh5/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/gyo63dh5/.hydra/hydra.yaml b/gyo63dh5/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44b5a18e5165005ac1931dd158b6409cacc6efd2 --- /dev/null +++ b/gyo63dh5/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/gyo63dh5 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/gyo63dh5/.hydra/overrides.yaml b/gyo63dh5/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/gyo63dh5/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/gyo63dh5/eval_policy.log b/gyo63dh5/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..556b92edebcf4333895248d9b1f8746c98100683 --- /dev/null +++ b/gyo63dh5/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 20:54:39,386][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:54:39,390][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 20:54:43,139][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_10499.pt b/snapshot_10499.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b325cd86af0dbe9ae08ed33677cbeccc1c776cd --- /dev/null +++ b/snapshot_10499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19d961421842c0d8da2d37e4068e61578b70a802fc4762bb346d51749b4f83c0 +size 910102870 diff --git a/snapshot_11749.pt b/snapshot_11749.pt new file mode 100644 index 0000000000000000000000000000000000000000..653211d20159a6415aa04e67750d11cd493feefe --- /dev/null +++ b/snapshot_11749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61730bf00a9225e39b029ee114b3f1c4c5ff52b0797c43f8a6e9b6486bf63d9b +size 910102870 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..25c9a36124bc8d0094add65da871e8a975aa6962 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae39750d15054f1ae51c85d4ee9160c6c80840e8881c8f85b2e963cade940ad8 +size 910102870 diff --git a/snapshot_13101.pt b/snapshot_13101.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9cb9402bcdb31c1a2450e5a05a11913737d0786 --- /dev/null +++ b/snapshot_13101.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92ad65dd0955eb912733b63b9d84156c52501afcb40d644bb917148d1e5777d0 +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8f0bd0b6fc271eae41f44f423056968bd8494d --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8729c108ad72940943af7fe8840e79d6d9fd17edc69a1daa06f961d22922a747 +size 910102870 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..d659db2c1aea921b8d8f20330aa3f952de986041 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9175aec2d0861d5f1650ec24fc56b5428ab78d1f1a76f1f1476ef615dd6eda5c +size 910102870 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..203b5164bed350314731aca93b8ed36c52774c79 --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a24570fd23b59163cadc2a606b00b697f84cbd1890a454e1dce60ae1dc227a3f +size 910102870 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..89bdd409d50f7100eb71279a0a0d92803125af3c --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c75aea2648250a8bcda9e1a3f66c6c22433746856511db79252bd1c66a455489 +size 910102360 diff --git a/snapshot_6611.pt b/snapshot_6611.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9c762046c30e87f9d10a06a5ab45bca36cee9c0 --- /dev/null +++ b/snapshot_6611.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03f36c27a5eeb34faa9a95ed66ef2dc61b1bf89a7b9f2b9994f9bcfbf2c3bc7 +size 910102360 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..bef8089c02bec2297b46a041ee901d28e53fd4b6 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bbc03c96f25bc91217d5dc4b6df82468beb1771d7248a37533f4462115b600d +size 910102870 diff --git a/snapshot_8249.pt b/snapshot_8249.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8a554da1d58b4f7bd0f7ee0f4e3bf2ba194053e --- /dev/null +++ b/snapshot_8249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6176fc634dd12689367010a11bb4f3933b9b594367014e38516de72bcc1bdf +size 910102870 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c44ae0398d789e43a68d9d3fbd6454c637583495 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc565d4089698b81a716d54d28f64cf3763e71d39b314401f39ad341740348aa +size 910102870 diff --git a/you6ychq/.hydra/config.yaml b/you6ychq/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/you6ychq/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/you6ychq/.hydra/hydra.yaml b/you6ychq/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..198d3ae8f86164ecdc16fe4cb8e5beae347b780b --- /dev/null +++ b/you6ychq/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/you6ychq/.hydra/overrides.yaml b/you6ychq/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/you6ychq/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/you6ychq/episode_rosbags/aligned_depth_to_color_K.npy b/you6ychq/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/you6ychq/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/you6ychq/episode_rosbags/cam_tf_world.npy b/you6ychq/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/you6ychq/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/you6ychq/episode_rosbags/color_K.npy b/you6ychq/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/you6ychq/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/you6ychq/episode_rosbags/depth_K.npy b/you6ychq/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/you6ychq/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/you6ychq/episode_rosbags/episode_0_2025-01-07-17-41-33.bag b/you6ychq/episode_rosbags/episode_0_2025-01-07-17-41-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..ed8ac943f45935068cdadb0dd711f921f28f20c6 --- /dev/null +++ b/you6ychq/episode_rosbags/episode_0_2025-01-07-17-41-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79778df3c75074f1493fcdc81d7e4712d341c150a752e42a35424a9beaf47b88 +size 1411323144 diff --git a/you6ychq/episode_rosbags/episode_2_2025-01-07-17-43-04.bag b/you6ychq/episode_rosbags/episode_2_2025-01-07-17-43-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..db8082f3725508b7f166b5647ab9e88817ec1044 --- /dev/null +++ b/you6ychq/episode_rosbags/episode_2_2025-01-07-17-43-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2869f52725adc29e5451390c8a9ba74571427a50899b6c249fde2b39955673c0 +size 1414557376 diff --git a/you6ychq/episode_rosbags/episode_3_2025-01-07-17-43-50.bag b/you6ychq/episode_rosbags/episode_3_2025-01-07-17-43-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..b0d4ffec32d4b3cd5b916d4c958eb2ec26a226f2 --- /dev/null +++ b/you6ychq/episode_rosbags/episode_3_2025-01-07-17-43-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2447544a4555066f8aae0f8cc1bb2e2d186dee258f206cade71dbdcd578469 +size 1412716436 diff --git a/you6ychq/episode_rosbags/episode_4_2025-01-07-17-44-32.bag b/you6ychq/episode_rosbags/episode_4_2025-01-07-17-44-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..44f022be481175a873ea87864e18c166f791691b --- /dev/null +++ b/you6ychq/episode_rosbags/episode_4_2025-01-07-17-44-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3720d872c5749fc8a04712d579a673074df75d9358ad739123581f4c1f2f1e8 +size 1413127202 diff --git a/you6ychq/eval_robot.log b/you6ychq/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..effff038fd7fc2c3862f19aa9e607a8c1d534936 --- /dev/null +++ b/you6ychq/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:41:05,253][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:41:05,256][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/you6ychq/eval_video/0_eval.mp4 b/you6ychq/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9eecf086ad0b8232d4b1194d3f10fec800fb8551 --- /dev/null +++ b/you6ychq/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9dd74fcc70d62e9bbe0c85ac7894f5d05a339cee2b1235fad300eb0d8563e6e +size 1144320 diff --git a/you6ychq/eval_video/1_eval.mp4 b/you6ychq/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fcffc0e214169f991fba461490c5f795420a585 --- /dev/null +++ b/you6ychq/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:817e91198c35c162a7b6388ba03b1d3416ec9c989d55c6bf7af154bfd0df6f1a +size 1141097 diff --git a/you6ychq/eval_video/2_eval.mp4 b/you6ychq/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15b456aeebf3008fafcce053e616780b6b16f687 --- /dev/null +++ b/you6ychq/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999cf175c5611441285096d308afef64de3e0b2c187af42ffde89ffffbf283d5 +size 1137004 diff --git a/you6ychq/eval_video/3_eval.mp4 b/you6ychq/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4ec0f4968b0d3d7da151eb98c404d8d68bf6f66 --- /dev/null +++ b/you6ychq/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6c04d39b507ee63e063aaaf9b55a46e1a034e0e571e4fdcef82ddbb778776a +size 1141074 diff --git a/you6ychq/eval_video/4_eval.mp4 b/you6ychq/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd1b2600dfe751db1705bd1076c6bdedb2395adc --- /dev/null +++ b/you6ychq/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09bc5e6ad0d0988acb31d12a2c859c24abc25642635d83fe30b7434a9593b45a +size 1141214 diff --git a/you6ychq/tb/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 b/you6ychq/tb/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 new file mode 100644 index 0000000000000000000000000000000000000000..ca7fc16e2464587d3a78cbd611ad6ae745117d02 --- /dev/null +++ b/you6ychq/tb/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7bdff075ffb7de2de0812105e5dba3ec79f5596141e3a4c2416fb0a749c7e6c +size 6539 diff --git a/you6ychq/wandb/debug-internal.log b/you6ychq/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..09d16698d12087bdd6a4c2c1286fe12166c9cd53 --- /dev/null +++ b/you6ychq/wandb/debug-internal.log @@ -0,0 +1,735 @@ +2025-01-07 17:41:11,073 INFO StreamThr :2201506 [internal.py:wandb_internal():85] W&B internal server running at pid: 2201506, started at: 2025-01-07 17:41:11.073069 +2025-01-07 17:41:11,074 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:41:11,086 INFO WriterThread:2201506 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb +2025-01-07 17:41:11,086 DEBUG SenderThread:2201506 [sender.py:send():379] send: header +2025-01-07 17:41:11,095 DEBUG SenderThread:2201506 [sender.py:send():379] send: run +2025-01-07 17:41:12,353 INFO SenderThread:2201506 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files +2025-01-07 17:41:12,353 INFO SenderThread:2201506 [sender.py:_start_run_threads():1188] run started: you6ychq with start time 1736289671.073577 +2025-01-07 17:41:12,371 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:41:12,371 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:41:12,442 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:41:12,447 DEBUG HandlerThread:2201506 [system_info.py:__init__():26] System info init +2025-01-07 17:41:12,447 DEBUG HandlerThread:2201506 [system_info.py:__init__():41] System info init done +2025-01-07 17:41:12,447 INFO HandlerThread:2201506 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:41:12,447 INFO SystemMonitor:2201506 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:41:12,447 INFO HandlerThread:2201506 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:41:12,448 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:41:12,448 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:41:12,449 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:41:12,449 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:41:12,450 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:41:12,460 DEBUG HandlerThread:2201506 [system_info.py:probe():152] Probing system +2025-01-07 17:41:12,461 DEBUG HandlerThread:2201506 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:probe():200] Probing system done +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:41:12.460572', 'startedAt': '2025-01-07T22:41:11.057653', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 4499.954, 'min': 800.0, 'max': 4700.0}, {'current': 4500.008, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 769.9485206604004}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:41:12,466 INFO HandlerThread:2201506 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:41:12,466 INFO HandlerThread:2201506 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:_save_code():44] Saving code +2025-01-07 17:41:12,472 DEBUG HandlerThread:2201506 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:41:12,472 DEBUG HandlerThread:2201506 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:41:12,537 DEBUG HandlerThread:2201506 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:41:12,538 INFO HandlerThread:2201506 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:41:12,541 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:41:12,776 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:41:12,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:41:12,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:12,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:12,780 INFO HandlerThread:2201506 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/tb" + save: true +} +control { + relay_id: "12c98893ea374f7aaf99019a1fd4c21a" +} +_info { + stream_id: "you6ychq" +} + +2025-01-07 17:41:12,780 DEBUG HandlerThread:2201506 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:41:12,781 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/3pylbhxc-wandb-metadata.json +2025-01-07 17:41:12,845 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/jms3xgfv-diff.patch +2025-01-07 17:41:12,852 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/7bm9mgli-code/FISH/eval_robot.py +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: telemetry +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: tbrecord +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:41:12,876 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 with policy live +2025-01-07 17:41:13,070 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/4mk44jie-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch +2025-01-07 17:41:13,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH +2025-01-07 17:41:13,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code +2025-01-07 17:41:13,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:14,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:15,354 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:15,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:16,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:16,878 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:17,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:18,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:19,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:20,639 DEBUG SenderThread:2201506 [sender.py:send():379] send: config +2025-01-07 17:41:20,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:21,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:22,689 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:22,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:23,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:23,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:24,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:25,357 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:25,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:26,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:27,358 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:27,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:27,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:27,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:27,913 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:28,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:29,358 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:29,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:30,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:31,359 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:31,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:32,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:33,346 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:33,360 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:33,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:34,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:35,360 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:35,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:36,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:37,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:38,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:38,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:39,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:40,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:41,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:42,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:42,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:42,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:43,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:43,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:44,363 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:41:44,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:45,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:46,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:47,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:48,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:49,155 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:49,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:50,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:51,364 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:51,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:52,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:53,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:54,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:54,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:55,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:56,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:57,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:57,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:57,825 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:58,785 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:59,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:59,870 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:00,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:01,296 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:42:01,296 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 with policy now +2025-01-07 17:42:01,297 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:42:01,298 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:42:01,298 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:42:01,298 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:42:01,367 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos +2025-01-07 17:42:01,565 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/5txo3ami-media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:01,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:01,842 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/narutocu-media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:02,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:03,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:03,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:04,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:05,005 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:05,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:06,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:07,369 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:07,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:08,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:09,370 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:09,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:10,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:10,806 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:11,371 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:42:11,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:12,450 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:42:12,451 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:42:12,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:12,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:13,372 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:13,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:14,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:15,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:16,329 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:16,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:17,373 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:17,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:18,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:19,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:20,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:21,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:21,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:22,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:23,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:24,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:25,376 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:25,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:26,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:26,922 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:27,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:27,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:27,825 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:28,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:29,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:30,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:31,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:32,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:32,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:33,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:34,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:35,379 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:35,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:36,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:37,379 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:37,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:37,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:38,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:39,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:40,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:41,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:42,455 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:42:42,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:42,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:42,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:42,866 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:43,381 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:43,544 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:42:43,544 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 with policy now +2025-01-07 17:42:43,547 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:42:43,547 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:42:43,548 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:42:43,548 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:42:43,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:43,805 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/2dkgrhom-media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:42:44,682 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/zfrdp6qj-media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,798 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:45,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:45,799 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:46,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:47,802 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:48,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:48,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:49,384 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:49,806 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:50,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:51,384 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:51,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:52,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:53,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:54,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:54,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:55,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:56,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:57,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:57,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:57,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:58,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:59,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:59,922 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:00,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:01,214 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:01,215 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:01,215 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:01,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:01,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:01,662 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/cz2by9io-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:01,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:02,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:03,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:03,816 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:04,813 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:05,389 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:05,762 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:05,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:06,817 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:07,390 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:07,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:08,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:09,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:10,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:11,391 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:11,749 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:11,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:12,452 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:43:12,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:12,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:13,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:14,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:15,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:16,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:16,901 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:17,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:18,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:19,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:20,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:21,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:22,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:22,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:23,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:24,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:25,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:26,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:27,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:27,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:27,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:27,852 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:28,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:29,822 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:30,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:31,826 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:32,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:32,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:43:32,846 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_999cf175c56114412850.mp4 with policy now +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:32,848 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:32,893 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:33,105 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/w7j2nt6x-media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,399 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:43:33,731 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/vqrkfokf-media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,826 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:34,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:35,399 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:35,830 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:36,830 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:37,841 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:38,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:38,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:39,400 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:39,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:40,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:41,401 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:41,838 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:42,452 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:43:42,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:42,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:42,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:43,832 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:43,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:44,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:45,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:46,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:47,050 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:47,050 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:47,051 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:47,403 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:47,403 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:47,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:48,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:49,404 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:49,792 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:49,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:50,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:51,405 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:51,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:52,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:53,405 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:53,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:54,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:55,789 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:55,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:56,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:57,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:57,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:57,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:58,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:59,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:00,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:00,858 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:01,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:02,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:03,409 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:03,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:04,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:05,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:06,146 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:06,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:07,410 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:07,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:08,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:09,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:10,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:11,791 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:11,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:12,453 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:44:12,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:12,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:13,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:14,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:15,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:16,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:16,896 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:17,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:18,814 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:44:18,814 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 with policy now +2025-01-07 17:44:18,814 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:44:18,815 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:44:18,815 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:44:18,819 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:44:18,839 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:19,083 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/vty3lh3k-media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:44:19,726 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/bqu0ilyb-media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:20,847 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:21,415 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:21,847 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:22,552 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:22,856 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:23,415 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:23,848 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:24,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:25,854 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:26,853 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:27,417 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:27,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:27,782 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:27,855 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:27,934 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:28,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:29,103 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:44:29,103 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:44:29,105 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:44:29,417 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:44:29,418 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:44:29,687 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/lwk40kkv-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:44:29,897 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:30,861 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:31,418 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:31,868 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:32,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:32,937 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:33,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:34,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:35,420 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:35,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:36,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:37,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:38,798 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:38,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:39,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:40,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:41,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:42,454 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:44:42,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:42,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:42,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:43,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:43,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:44,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:45,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:46,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:47,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:48,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:49,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:49,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:50,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:51,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:52,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:53,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:54,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:54,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:55,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:56,866 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:57,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:57,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:57,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:58,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:59,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:59,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:00,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:01,097 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:01,097 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 with policy now +2025-01-07 17:45:01,101 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:01,102 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:01,102 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:01,103 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:01,357 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/w9unkm6b-media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:45:01,703 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/sis5wbn8-media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:02,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:03,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:03,874 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:04,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:04,878 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:05,882 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:06,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:07,431 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:07,879 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:08,883 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:09,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:09,880 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:10,061 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:10,152 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:10,152 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:10,153 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:10,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:10,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:10,568 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:45:10,568 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:45:10,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:10,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:11,034 INFO wandb-upload_1:2201506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1yqpfzdl +2025-01-07 17:45:11,216 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:45:11,225 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media +2025-01-07 17:45:11,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval +2025-01-07 17:45:11,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table +2025-01-07 17:45:11,446 INFO SenderThread:2201506 [sender.py:send_request_log_artifact():1525] logged artifact run-you6ychq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk5MDEyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxODg1NQ==', 'latestArtifact': None}} +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:11,446 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,447 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send():379] send: telemetry +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,447 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,448 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,448 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,449 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,449 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,450 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,450 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,450 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,450 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,451 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,451 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,452 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,452 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:45:11,634 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/6tsaj4d8-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:11,665 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/95aelmml-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:11,783 INFO wandb-upload_1:2201506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplhmw4f05 +2025-01-07 17:45:11,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:11,933 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/oo6ga3vj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:12,160 INFO SenderThread:2201506 [sender.py:send_request_log_artifact():1525] logged artifact run-you6ychq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk5MDIxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxODkyMA==', 'latestArtifact': None}} +2025-01-07 17:45:12,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:12,160 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json with policy now +2025-01-07 17:45:12,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:12,161 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:12,161 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:12,407 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/ssdkifkk-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:12,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:12,454 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:45:12,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:45:12,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:45:12,868 DEBUG SenderThread:2201506 [sender.py:send():379] send: exit +2025-01-07 17:45:12,868 INFO SenderThread:2201506 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:45:12,868 INFO SenderThread:2201506 [sender.py:send_exit():588] handling runtime: 240 +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:send_exit():594] send defer +2025-01-07 17:45:12,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,869 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:45:12,869 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:45:12,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,869 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:45:12,870 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,870 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:45:12,870 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:45:12,870 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:45:12,871 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:45:12,871 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:45:12,871 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:45:12,921 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:45:12,922 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:45:12,922 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:45:12,922 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,922 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:45:12,922 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:45:12,922 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:45:12,923 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,923 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:45:12,923 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,923 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:45:12,923 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:45:12,923 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,923 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:45:13,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:13,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:15,435 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:19,158 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:19,159 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:19,159 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,160 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:45:19,160 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:19,160 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,160 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:19,161 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:19,161 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:19,161 DEBUG SenderThread:2201506 [sender.py:send():379] send: summary +2025-01-07 17:45:19,165 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:19,165 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:45:19,166 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,166 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:45:19,166 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:45:19,169 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:19,270 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:45:19,270 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,270 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:45:19,270 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,270 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:45:19,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:45:19,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:19,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:21,361 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:45:21,361 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:21,361 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:21,361 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:45:21,362 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:21,362 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:45:21,362 INFO SenderThread:2201506 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:45:21,364 INFO SenderThread:2201506 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:45:21,366 INFO SenderThread:2201506 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:45:21,380 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:45:21,380 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:21,380 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:45:21,381 DEBUG SenderThread:2201506 [sender.py:send():379] send: artifact +2025-01-07 17:45:21,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:21,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:45:21,882 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml config.yaml +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json wandb-summary.json +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt requirements.txt +2025-01-07 17:45:22,439 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:45:22,439 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:22,442 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log output.log +2025-01-07 17:45:22,442 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch diff.patch +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:45:22,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:22,448 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:22,448 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:45:22,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:45:22,613 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:45:22,622 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:22,636 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt +2025-01-07 17:45:22,655 INFO wandb-upload_4:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:22,799 INFO wandb-upload_3:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/qs52s3u7-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:22,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:22,828 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,000 INFO Thread-11 (_thread_body):2201506 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:45:23,000 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,000 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:45:23,000 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,000 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:45:23,000 INFO SenderThread:2201506 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:45:23,001 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:45:23,002 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,002 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:45:23,002 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,002 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:45:23,002 INFO SenderThread:2201506 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:45:23,398 INFO SenderThread:2201506 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:45:23,398 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:45:23,399 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,399 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,399 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:45:23,399 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send():379] send: final +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send():379] send: footer +2025-01-07 17:45:23,399 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,399 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:45:23,400 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,400 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:45:23,400 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:23,400 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,400 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:45:23,401 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:45:23,402 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:23,402 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:45:23,403 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:45:23,403 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:23,465 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,465 INFO MainThread:2201506 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:45:23,465 INFO MainThread:2201506 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:45:23,466 INFO MainThread:2201506 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:45:23,466 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:45:23,466 INFO HandlerThread:2201506 [handler.py:finish():882] shutting down handler +2025-01-07 17:45:24,402 INFO WriterThread:2201506 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [sender.py:finish():1615] shutting down sender +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [file_pusher.py:join():175] waiting for file pusher diff --git a/you6ychq/wandb/debug.log b/you6ychq/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..61e8c7bfec1e55843343cffbc15f6ba443330c1a --- /dev/null +++ b/you6ychq/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:41:11,064 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Configure stats pid to 2201339 +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/settings +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug.log +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug-internal.log +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq'} +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():619] starting backend +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():623] setting up manager +2025-01-07 17:41:11,072 INFO MainThread:2201339 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:41:11,073 INFO MainThread:2201339 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:41:11,085 INFO MainThread:2201339 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:41:11,091 INFO MainThread:2201339 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:41:12,371 INFO MainThread:2201339 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:41:12,437 INFO MainThread:2201339 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:41:12,437 INFO MainThread:2201339 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:41:12,779 INFO MainThread:2201339 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:41:12,780 INFO MainThread:2201339 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/tb, True +2025-01-07 17:41:20,638 INFO MainThread:2201339 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:45:24,466 WARNING MsgRouterThr:2201339 [router.py:message_loop():77] message_loop has been closed diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py b/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml b/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fd3bf54d00bb069b042b34eaba38fdde99a00ac --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736289671 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch b/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1af31b2ff07685aba62f0764c3073e6e9867fe4d --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch @@ -0,0 +1,21 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..6e694c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..050f75761d7b6273f3088165b89fcba85be80dd0 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[30, 55, 0.5454545454545454, 0.386904776118963, 0.6963961869085433, 0.4168242216276581, 0.6795179396508717, 0.4371866434938173, 0.6503013074097438, 0.45355755091642347, 0.6361154354836506]]} \ No newline at end of file diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9eecf086ad0b8232d4b1194d3f10fec800fb8551 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9dd74fcc70d62e9bbe0c85ac7894f5d05a339cee2b1235fad300eb0d8563e6e +size 1144320 diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7fcffc0e214169f991fba461490c5f795420a585 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:817e91198c35c162a7b6388ba03b1d3416ec9c989d55c6bf7af154bfd0df6f1a +size 1141097 diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15b456aeebf3008fafcce053e616780b6b16f687 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999cf175c5611441285096d308afef64de3e0b2c187af42ffde89ffffbf283d5 +size 1137004 diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4ec0f4968b0d3d7da151eb98c404d8d68bf6f66 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6c04d39b507ee63e063aaaf9b55a46e1a034e0e571e4fdcef82ddbb778776a +size 1141074 diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd1b2600dfe751db1705bd1076c6bdedb2395adc --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09bc5e6ad0d0988acb31d12a2c859c24abc25642635d83fe30b7434a9593b45a +size 1141214 diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log b/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cc8f5ba09af8dec1cf678738a510979606362e53 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736289680.639508]: resetting environment +[INFO] [1736289680.642989]: cleared current plan +[INFO] [1736289680.644010]: moving to home +[INFO] [1736289681.646269]: reached home +[INFO] [1736289681.647483]: reset action history +[INFO] [1736289684.356410]: environment reset +Starting episode 0 +[INFO] [1736289684.360855]: resetting environment +[INFO] [1736289684.367213]: cleared current plan +[INFO] [1736289684.368721]: moving to home +[INFO] [1736289685.371223]: reached home +[INFO] [1736289685.372487]: reset action history +[INFO] [1736289688.081468]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736289689.631041]: resetting environment +[INFO] [1736289689.634029]: cleared current plan +[INFO] [1736289689.635652]: moving to home +[INFO] [1736289690.637262]: reached home +[INFO] [1736289690.638526]: reset action history +[INFO] [1736289693.345818]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/episode_rosbags/episode_0_2025-01-07-17-41-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2201339.log +started roslaunch server http://158.130.50.37:34279/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2201605] +process[print_text-2]: started with pid [2201606] +process[rosbag_record-3]: started with pid [2201630] +started bagging! +For topic gripper_width: timestamp difference is 101356796 for nearest: 1736289709145494035 - target: 1736289709044137239 at idx 236 +[INFO] [1736289720.328595]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736289721.079159]: Stored episode 1. +[INFO] [1736289721.079561]: Saving video... +[INFO] [1736289721.277047]: Video saved! +Starting episode 1 +[INFO] [1736289721.297688]: resetting environment +[INFO] [1736289721.302421]: cleared current plan +[INFO] [1736289721.302823]: moving to home +[INFO] [1736289725.005262]: reached home +[INFO] [1736289725.005748]: reset action history +[INFO] [1736289727.764189]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289731.330846]: resetting environment +[INFO] [1736289731.333321]: cleared current plan +[INFO] [1736289731.333618]: moving to home +[INFO] [1736289732.335268]: reached home +[INFO] [1736289732.335885]: reset action history +[INFO] [1736289735.092302]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/episode_rosbags/episode_1_2025-01-07-17-42-15.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2201339.log +started roslaunch server http://158.130.50.37:42707/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2201758] +process[print_text-5]: started with pid [2201759] +process[rosbag_record-6]: started with pid [2201760] +started bagging! +For topic gripper_width: timestamp difference is -104726643 for nearest: 1736289742678827196 - target: 1736289742783553839 at idx 101 +For topic gripper_width: timestamp difference is 103598918 for nearest: 1736289753278826322 - target: 1736289753175227404 at idx 239 +For topic gripper_width: timestamp difference is 105399738 for nearest: 1736289755678825031 - target: 1736289755573425293 at idx 249 +[INFO] [1736289762.041375]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736289762.816058]: Stored episode 2. +[INFO] [1736289762.818800]: Saving video... +[INFO] [1736289763.521602]: Video saved! +Starting episode 2 +[INFO] [1736289763.552215]: resetting environment +[INFO] [1736289763.567697]: cleared current plan +[INFO] [1736289763.570369]: moving to home +[WARN] [1736289763.201830]: Plan exhausted +[WARN] [1736289763.241827]: Plan exhausted +[WARN] [1736289763.283090]: Plan exhausted +[WARN] [1736289763.321796]: Plan exhausted +[WARN] [1736289763.362707]: Plan exhausted +[WARN] [1736289763.401809]: Plan exhausted +[WARN] [1736289763.441999]: Plan exhausted +[WARN] [1736289763.482615]: Plan exhausted +[WARN] [1736289763.523016]: Plan exhausted +[WARN] [1736289763.561769]: Plan exhausted +[INFO] [1736289767.274842]: reached home +[INFO] [1736289767.280118]: reset action history +[INFO] [1736289770.052350]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289780.779902]: resetting environment +[INFO] [1736289780.787171]: cleared current plan +[INFO] [1736289780.792184]: moving to home +[INFO] [1736289781.794813]: reached home +[INFO] [1736289781.795337]: reset action history +[INFO] [1736289784.504849]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/episode_rosbags/episode_2_2025-01-07-17-43-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2201339.log +started roslaunch server http://158.130.50.37:45279/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2201907] +process[print_text-8]: started with pid [2201908] +process[rosbag_record-9]: started with pid [2201909] +started bagging! +For topic gripper_width: timestamp difference is -136641760 for nearest: 1736289789478813152 - target: 1736289789615454912 at idx 81 +[INFO] [1736289811.464359]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit[rosbag_record-9] killing on exit +[INFO] [1736289812.261676]: Stored episode 3. +[INFO] [1736289812.262499]: Saving video... +[INFO] [1736289812.812469]: Video saved! +Starting episode 3 +[INFO] [1736289812.845698]: resetting environment +[INFO] [1736289812.872499]: cleared current plan +[INFO] [1736289812.874233]: moving to home +[WARN] [1736289812.801912]: Plan exhausted +[WARN] [1736289812.843484]: Plan exhausted +[INFO] [1736289816.581829]: reached home +[INFO] [1736289816.584613]: reset action history +[INFO] [1736289819.382499]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289826.612988]: resetting environment +[INFO] [1736289826.627335]: cleared current plan +[INFO] [1736289826.630454]: moving to home +[INFO] [1736289827.634784]: reached home +[INFO] [1736289827.641678]: reset action history +[INFO] [1736289830.553910]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/episode_rosbags/episode_3_2025-01-07-17-43-50.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2201339.log +started roslaunch server http://158.130.50.37:44021/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2202067] +process[print_text-11]: started with pid [2202068] +process[rosbag_record-12]: started with pid [2202069] +started bagging! +For topic gripper_width: timestamp difference is -258589928 for nearest: 1736289840978827531 - target: 1736289841237417459 at idx 145 +For topic gripper_width: timestamp difference is 111912901 for nearest: 1736289846145492966 - target: 1736289846033580065 at idx 216 +[INFO] [1736289857.701477]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736289858.485495]: Stored episode 4. +[INFO] [1736289858.490217]: Saving video... +[INFO] [1736289858.775869]: Video saved! +Starting episode 4 +[INFO] [1736289858.818428]: resetting environment +[INFO] [1736289858.844739]: cleared current plan +[INFO] [1736289858.845762]: moving to home +[INFO] [1736289862.549790]: reached home +[INFO] [1736289862.557095]: reset action history +[INFO] [1736289865.285224]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289868.831609]: resetting environment +[INFO] [1736289868.847386]: cleared current plan +[INFO] [1736289868.847603]: moving to home +[INFO] [1736289869.849460]: reached home +[INFO] [1736289869.852988]: reset action history +[INFO] [1736289872.630886]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/episode_rosbags/episode_4_2025-01-07-17-44-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2201339.log +started roslaunch server http://158.130.50.37:46635/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2202212] +process[print_text-14]: started with pid [2202213] +process[rosbag_record-15]: started with pid [2202214] +started bagging! +For topic gripper_width: timestamp difference is 101342211 for nearest: 1736289874045495520 - target: 1736289873944153309 at idx 24 +[INFO] [1736289899.830562]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736289900.620168]: Stored episode 5. +[INFO] [1736289900.628906]: Saving video... +[INFO] [1736289901.054718]: Video saved! +[INFO] [1736289901.102581]: resetting environment +[INFO] [1736289901.131544]: cleared current plan +[INFO] [1736289901.131792]: moving to home +[INFO] [1736289904.833579]: reached home +[INFO] [1736289904.838322]: reset action history +[INFO] [1736289907.653702]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt b/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json b/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..16e8500e3154ef57dd7b593412357deee39d0ee9 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:41:12.460572", + "startedAt": "2025-01-07T22:41:11.057653", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.501, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.982, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.954, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.008, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 769.9485206604004 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json b/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ee89fa1334108365b18fb6ea097aa8d8cc18db16 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "d9dd74fcc70d62e9bbe0c85ac7894f5d05a339cee2b1235fad300eb0d8563e6e", "size": 1144320, "path": "media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4"}, "global_step": 664, "_timestamp": 1736289911.140666, "_runtime": 240.06708908081055, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "817e91198c35c162a7b6388ba03b1d3416ec9c989d55c6bf7af154bfd0df6f1a", "size": 1141097, "path": "media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4"}, "eval/num_success": 5.0, "episode": 55.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "999cf175c5611441285096d308afef64de3e0b2c187af42ffde89ffffbf283d5", "size": 1137004, "path": "media/videos/eval/2_eval_2_999cf175c56114412850.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9c6c04d39b507ee63e063aaaf9b55a46e1a034e0e571e4fdcef82ddbb778776a", "size": 1141074, "path": "media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "09bc5e6ad0d0988acb31d12a2c859c24abc25642635d83fe30b7434a9593b45a", "size": 1141214, "path": "media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://g6jegjflf5qz8xu1ldknrffir81qm28vvgco8fg0r67qqg8ym3c3314shy3p7rl9mj0tjwi3kh9bm1zs7k7wj725gwtrp1s448vb24xxgw2o1nsm3a9su7s8oen546qe/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://we4v9zttvi2m0d638tzu30w1itt4wogdyeomu9fwsyghgaqmcu9supt80e50rg2ch7bplz5wgheij7716kvovvep2oym4hy6uov2to0aobjffsxj8e5b34ic9iyq8uwc:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 30, "total_num_episodes": 55, "total_success_rate": 0.5454545454545454, "total_success_rate_lb_0.01": 0.386904776118963, "total_success_rate_ub_0.01": 0.6963961869085433, "total_success_rate_lb_0.025": 0.4168242216276581, "total_success_rate_ub_0.025": 0.6795179396508717, "total_success_rate_lb_0.05": 0.4371866434938173, "total_success_rate_ub_0.05": 0.6503013074097438, "total_success_rate_lb_0.1": 0.45355755091642347, "total_success_rate_ub_0.1": 0.6361154354836506, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "da430633f673220357fdf073c1e35553cfb9baaeba00dbe4f8caf7a3a0de7f1c", "size": 520, "artifact_path": "wandb-client-artifact://hf8vh63im8hwoym600jmf2hyem51x7x15uwn90dl1n4oi5drmx2x7tjg9iwitzn29f5n4aaacc81ijmifi62zwlptd1cpasycg7skcdxdk2qs3i5bg46srp6ancfnhmn/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ozkfo7e1f6ubq69znxu9tqopskiai17g8z5ssnydvtmez8opr0dg7oasxwcb42hzzygi8nzxqf356li6guxpn2pi0boxk6sokq8u7ypm1orbwcjqcm1es1g3dqhubdxf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 240}, "eval/total_success_rate": 0.5454545617103577, "eval/total_success_rate_lb_0.01": 0.386904776096344, "eval/total_success_rate_ub_0.01": 0.6963961720466614, "eval/total_success_rate_lb_0.025": 0.41682422161102295, "eval/total_success_rate_ub_0.025": 0.679517924785614, "eval/total_success_rate_lb_0.05": 0.43718665838241577, "eval/total_success_rate_ub_0.05": 0.6503012776374817, "eval/total_success_rate_lb_0.1": 0.453557550907135, "eval/total_success_rate_ub_0.1": 0.6361154317855835} \ No newline at end of file diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug-internal.log b/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..09d16698d12087bdd6a4c2c1286fe12166c9cd53 --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug-internal.log @@ -0,0 +1,735 @@ +2025-01-07 17:41:11,073 INFO StreamThr :2201506 [internal.py:wandb_internal():85] W&B internal server running at pid: 2201506, started at: 2025-01-07 17:41:11.073069 +2025-01-07 17:41:11,074 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:41:11,086 INFO WriterThread:2201506 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb +2025-01-07 17:41:11,086 DEBUG SenderThread:2201506 [sender.py:send():379] send: header +2025-01-07 17:41:11,095 DEBUG SenderThread:2201506 [sender.py:send():379] send: run +2025-01-07 17:41:12,353 INFO SenderThread:2201506 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files +2025-01-07 17:41:12,353 INFO SenderThread:2201506 [sender.py:_start_run_threads():1188] run started: you6ychq with start time 1736289671.073577 +2025-01-07 17:41:12,371 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:41:12,371 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:41:12,442 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:41:12,447 DEBUG HandlerThread:2201506 [system_info.py:__init__():26] System info init +2025-01-07 17:41:12,447 DEBUG HandlerThread:2201506 [system_info.py:__init__():41] System info init done +2025-01-07 17:41:12,447 INFO HandlerThread:2201506 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:41:12,447 INFO SystemMonitor:2201506 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:41:12,447 INFO HandlerThread:2201506 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:41:12,448 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:41:12,448 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:41:12,449 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:41:12,449 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:41:12,450 INFO SystemMonitor:2201506 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:41:12,460 DEBUG HandlerThread:2201506 [system_info.py:probe():152] Probing system +2025-01-07 17:41:12,461 DEBUG HandlerThread:2201506 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:probe():200] Probing system done +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:41:12.460572', 'startedAt': '2025-01-07T22:41:11.057653', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.501, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.982, 'min': 800.0, 'max': 4700.0}, {'current': 4499.954, 'min': 800.0, 'max': 4700.0}, {'current': 4500.008, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 769.9485206604004}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:41:12,466 INFO HandlerThread:2201506 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:41:12,466 INFO HandlerThread:2201506 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:41:12,466 DEBUG HandlerThread:2201506 [system_info.py:_save_code():44] Saving code +2025-01-07 17:41:12,472 DEBUG HandlerThread:2201506 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:41:12,472 DEBUG HandlerThread:2201506 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:41:12,537 DEBUG HandlerThread:2201506 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:41:12,538 INFO HandlerThread:2201506 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:41:12,541 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:41:12,541 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:41:12,776 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:41:12,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:41:12,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:12,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:12,780 INFO HandlerThread:2201506 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/tb" + save: true +} +control { + relay_id: "12c98893ea374f7aaf99019a1fd4c21a" +} +_info { + stream_id: "you6ychq" +} + +2025-01-07 17:41:12,780 DEBUG HandlerThread:2201506 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:41:12,781 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/3pylbhxc-wandb-metadata.json +2025-01-07 17:41:12,845 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/jms3xgfv-diff.patch +2025-01-07 17:41:12,852 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/7bm9mgli-code/FISH/eval_robot.py +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: telemetry +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: tbrecord +2025-01-07 17:41:12,876 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:41:12,876 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 with policy live +2025-01-07 17:41:13,070 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/4mk44jie-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:13,355 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch +2025-01-07 17:41:13,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH +2025-01-07 17:41:13,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code +2025-01-07 17:41:13,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:14,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:15,354 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:15,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:16,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:16,878 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:17,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:18,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:19,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:20,639 DEBUG SenderThread:2201506 [sender.py:send():379] send: config +2025-01-07 17:41:20,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:21,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:22,689 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:22,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:23,356 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:23,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:24,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:25,357 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:25,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:26,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:27,358 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:27,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:27,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:27,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:27,913 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:28,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:29,358 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:29,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:30,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:31,359 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:31,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:32,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:33,346 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:33,360 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:33,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:34,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:35,360 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:35,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:36,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:37,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:38,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:38,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:39,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:40,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:41,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:42,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:42,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:42,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:43,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:43,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:44,363 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:41:44,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:45,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:46,782 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:47,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:48,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:49,155 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:49,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:50,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:51,364 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:41:51,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:52,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:53,783 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:54,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:41:54,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:55,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:56,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:57,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:41:57,777 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:41:57,825 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:58,785 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:59,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:41:59,870 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:00,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:01,296 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:42:01,296 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 with policy now +2025-01-07 17:42:01,297 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:42:01,298 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:42:01,298 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:42:01,298 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:42:01,367 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media +2025-01-07 17:42:01,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos +2025-01-07 17:42:01,565 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/5txo3ami-media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:01,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:01,842 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/narutocu-media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:42:02,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:03,368 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:03,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:04,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:05,005 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:05,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:06,786 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:07,369 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:07,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:08,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:09,370 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:09,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:10,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:10,806 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:11,371 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:42:11,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:12,450 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:42:12,451 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:42:12,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:12,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:13,372 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:13,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:14,787 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:15,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:16,329 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:16,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:17,373 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:17,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:18,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:19,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:20,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:21,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:21,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:22,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:23,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:24,793 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:25,376 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:25,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:26,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:26,922 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:27,777 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:27,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:27,825 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:28,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:29,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:30,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:31,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:32,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:32,794 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:33,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:34,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:35,379 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:35,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:36,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:37,379 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:37,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:37,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:38,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:39,795 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:40,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:41,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:42,455 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:42:42,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:42,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:42,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:42,866 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:43,381 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:43,544 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:42:43,544 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 with policy now +2025-01-07 17:42:43,547 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:42:43,547 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:42:43,548 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:42:43,548 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:42:43,796 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:43,805 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/2dkgrhom-media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:42:44,682 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/zfrdp6qj-media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:42:44,798 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:45,382 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:45,799 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:46,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:47,802 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:48,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:48,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:49,384 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:49,806 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:50,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:51,384 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:42:51,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:52,805 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:53,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:54,780 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:42:54,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:55,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:56,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:57,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:42:57,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:42:57,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:58,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:59,807 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:42:59,922 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:00,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:01,214 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:01,215 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:01,215 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:01,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:01,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:01,662 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/cz2by9io-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:01,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:02,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:03,388 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:03,816 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:04,813 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:05,389 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:05,762 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:05,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:06,817 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:07,390 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:07,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:08,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:09,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:10,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:11,391 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:11,749 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:11,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:12,452 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:43:12,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:12,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:13,818 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:14,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:15,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:16,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:16,901 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:17,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:18,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:19,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:20,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:21,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:22,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:22,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:23,819 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:24,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:25,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:26,820 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:27,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:27,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:27,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:27,852 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:28,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:29,822 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:30,821 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:31,826 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:32,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:32,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:43:32,846 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_999cf175c56114412850.mp4 with policy now +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:32,846 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:32,848 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:32,893 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:33,105 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/w7j2nt6x-media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:33,398 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,399 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:43:33,731 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/vqrkfokf-media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:43:33,826 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:34,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:35,399 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:35,830 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:36,830 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:37,841 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:38,784 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:38,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:39,400 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:39,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:40,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:41,401 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:41,838 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:42,452 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:43:42,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:42,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:42,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:43,832 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:43,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:44,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:45,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:46,833 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:47,050 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:43:47,050 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:43:47,051 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:43:47,403 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:43:47,403 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:43:47,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:48,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:49,404 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:49,792 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:49,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:50,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:51,405 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:51,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:52,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:53,405 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:43:53,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:54,834 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:55,789 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:43:55,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:56,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:57,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:43:57,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:43:57,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:58,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:43:59,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:00,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:00,858 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:01,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:02,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:03,409 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:03,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:04,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:05,835 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:06,146 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:06,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:07,410 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:07,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:08,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:09,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:10,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:11,791 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:11,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:12,453 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:44:12,778 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:12,778 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:12,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:13,836 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:14,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:15,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:16,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:16,896 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:17,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:18,814 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:44:18,814 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 with policy now +2025-01-07 17:44:18,814 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:44:18,815 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:44:18,815 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:44:18,819 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:44:18,839 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:19,083 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/vty3lh3k-media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,414 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:44:19,726 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/bqu0ilyb-media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:44:19,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:20,847 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:21,415 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:21,847 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:22,552 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:22,856 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:23,415 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:23,848 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:24,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:25,854 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:26,853 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:27,417 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:27,781 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:27,782 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:27,855 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:27,934 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:28,850 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:29,103 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:44:29,103 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:44:29,105 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:44:29,417 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:44:29,418 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:44:29,687 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/lwk40kkv-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:44:29,897 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:30,861 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:31,418 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:31,868 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:32,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:32,937 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:33,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:34,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:35,420 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:44:35,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:36,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:37,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:38,798 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:38,862 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:39,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:40,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:41,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:42,454 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:44:42,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:42,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:42,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:43,845 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:43,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:44,863 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:45,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:46,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:47,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:48,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:49,808 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:49,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:50,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:51,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:52,864 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:53,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:54,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:54,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:55,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:56,866 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:57,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:44:57,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:44:57,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:58,865 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:44:59,831 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:44:59,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:00,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:01,097 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:01,097 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 with policy now +2025-01-07 17:45:01,101 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:01,102 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:01,102 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:01,103 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:01,357 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/w9unkm6b-media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval +2025-01-07 17:45:01,703 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/sis5wbn8-media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:01,873 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:02,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:03,429 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:03,874 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:04,837 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:04,878 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:05,882 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:06,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:07,431 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:07,879 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:08,883 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:09,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:09,880 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:10,061 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:10,152 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:10,152 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:10,153 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:10,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:10,432 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:10,568 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:45:10,568 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:45:10,809 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:10,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:11,034 INFO wandb-upload_1:2201506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1yqpfzdl +2025-01-07 17:45:11,216 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:45:11,225 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:11,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media +2025-01-07 17:45:11,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval +2025-01-07 17:45:11,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table +2025-01-07 17:45:11,446 INFO SenderThread:2201506 [sender.py:send_request_log_artifact():1525] logged artifact run-you6ychq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk5MDEyOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxODg1NQ==', 'latestArtifact': None}} +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:11,446 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:11,446 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,447 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send():379] send: telemetry +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,447 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,447 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,448 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,448 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,449 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,449 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,449 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,450 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,450 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,450 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,450 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,451 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,451 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,451 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:11,452 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:11,452 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:45:11,634 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/6tsaj4d8-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:11,665 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/95aelmml-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:11,783 INFO wandb-upload_1:2201506 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplhmw4f05 +2025-01-07 17:45:11,875 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:11,933 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/oo6ga3vj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:12,160 INFO SenderThread:2201506 [sender.py:send_request_log_artifact():1525] logged artifact run-you6ychq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk5MDIxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxODkyMA==', 'latestArtifact': None}} +2025-01-07 17:45:12,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: files +2025-01-07 17:45:12,160 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json with policy now +2025-01-07 17:45:12,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:12,161 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:12,161 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:12,407 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/ssdkifkk-media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:12,433 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:12,454 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:45:12,779 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:45:12,779 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:45:12,868 DEBUG SenderThread:2201506 [sender.py:send():379] send: exit +2025-01-07 17:45:12,868 INFO SenderThread:2201506 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:45:12,868 INFO SenderThread:2201506 [sender.py:send_exit():588] handling runtime: 240 +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:send_exit():594] send defer +2025-01-07 17:45:12,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,869 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:45:12,869 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:45:12,869 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:45:12,869 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,869 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:45:12,870 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,870 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:45:12,870 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:45:12,870 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:45:12,870 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:45:12,871 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:45:12,871 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:45:12,871 DEBUG SystemMonitor:2201506 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:45:12,921 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:45:12,922 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:45:12,922 INFO HandlerThread:2201506 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:45:12,922 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,922 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:45:12,922 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:45:12,922 DEBUG SenderThread:2201506 [sender.py:send():379] send: stats +2025-01-07 17:45:12,923 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,923 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:45:12,923 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:12,923 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:45:12,923 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:45:12,923 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:12,923 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:45:13,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:13,434 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:15,435 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:19,158 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:19,159 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:19,159 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,160 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:45:19,160 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:19,160 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,160 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:45:19,160 DEBUG SenderThread:2201506 [sender.py:send():379] send: history +2025-01-07 17:45:19,161 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:45:19,161 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:19,161 DEBUG SenderThread:2201506 [sender.py:send():379] send: summary +2025-01-07 17:45:19,165 INFO SenderThread:2201506 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:45:19,165 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:45:19,166 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,166 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:45:19,166 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,166 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:45:19,169 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:45:19,270 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:45:19,270 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:19,270 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:45:19,270 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:19,270 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:45:19,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:45:19,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:19,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:21,361 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:45:21,361 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:21,361 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:21,361 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:45:21,362 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:21,362 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:45:21,362 INFO SenderThread:2201506 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:45:21,364 INFO SenderThread:2201506 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:45:21,366 INFO SenderThread:2201506 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:45:21,380 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:45:21,380 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:21,380 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:45:21,381 DEBUG SenderThread:2201506 [sender.py:send():379] send: artifact +2025-01-07 17:45:21,437 INFO Thread-12 :2201506 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:21,827 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:45:21,882 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:45:21,882 INFO SenderThread:2201506 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml config.yaml +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json wandb-summary.json +2025-01-07 17:45:22,438 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt requirements.txt +2025-01-07 17:45:22,439 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:45:22,439 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:22,442 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log output.log +2025-01-07 17:45:22,442 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/diff.patch diff.patch +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 media/videos/eval/1_eval_1_817e91198c35c162a7b6.mp4 +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 media/videos/eval/4_eval_4_09bc5e6ad0d0988acb31.mp4 +2025-01-07 17:45:22,444 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 media/videos/eval/0_eval_0_d9dd74fcc70d62e9bbe0.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 media/videos/eval/3_eval_3_9c6c04d39b507ee63e06.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/videos/eval/2_eval_2_999cf175c56114412850.mp4 media/videos/eval/2_eval_2_999cf175c56114412850.mp4 +2025-01-07 17:45:22,447 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json media/table/eval/total_success_rate_ci_6_da430633f673220357fd.table.json +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:45:22,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:22,448 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:22,448 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:45:22,448 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:45:22,448 INFO SenderThread:2201506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:45:22,613 INFO wandb-upload_0:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/config.yaml +2025-01-07 17:45:22,622 INFO wandb-upload_1:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/wandb-summary.json +2025-01-07 17:45:22,636 INFO wandb-upload_2:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/requirements.txt +2025-01-07 17:45:22,655 INFO wandb-upload_4:2201506 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/files/output.log +2025-01-07 17:45:22,799 INFO wandb-upload_3:2201506 [upload_job.py:push():130] Uploaded file /tmp/tmphvtllm8lwandb/qs52s3u7-events.out.tfevents.1736289672.leonmkim-ROG-Strix-G15CS-G15CS.2201339.0 +2025-01-07 17:45:22,828 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:22,828 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,000 INFO Thread-11 (_thread_body):2201506 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:45:23,000 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,000 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:45:23,000 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,000 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:45:23,000 INFO SenderThread:2201506 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:45:23,001 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:45:23,002 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,002 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:45:23,002 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,002 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:45:23,002 INFO SenderThread:2201506 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:45:23,398 INFO SenderThread:2201506 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:45:23,398 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:45:23,399 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,399 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,399 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:45:23,399 INFO SenderThread:2201506 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send():379] send: final +2025-01-07 17:45:23,399 DEBUG SenderThread:2201506 [sender.py:send():379] send: footer +2025-01-07 17:45:23,399 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:45:23,399 INFO HandlerThread:2201506 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:45:23,400 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: defer +2025-01-07 17:45:23,400 INFO SenderThread:2201506 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:45:23,400 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:23,400 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,400 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:45:23,401 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:45:23,402 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:45:23,402 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:45:23,403 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:45:23,403 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:45:23,465 DEBUG SenderThread:2201506 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:45:23,465 INFO MainThread:2201506 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:45:23,465 INFO MainThread:2201506 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:45:23,466 INFO MainThread:2201506 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:45:23,466 DEBUG HandlerThread:2201506 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:45:23,466 INFO HandlerThread:2201506 [handler.py:finish():882] shutting down handler +2025-01-07 17:45:24,402 INFO WriterThread:2201506 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [sender.py:finish():1615] shutting down sender +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:45:24,465 INFO SenderThread:2201506 [file_pusher.py:join():175] waiting for file pusher diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug.log b/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..61e8c7bfec1e55843343cffbc15f6ba443330c1a --- /dev/null +++ b/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:41:11,064 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Configure stats pid to 2201339 +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/settings +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:41:11,065 INFO MainThread:2201339 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug.log +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/wandb/run-20250107_174111-you6ychq/logs/debug-internal.log +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq'} +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():619] starting backend +2025-01-07 17:41:11,066 INFO MainThread:2201339 [wandb_init.py:init():623] setting up manager +2025-01-07 17:41:11,072 INFO MainThread:2201339 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:41:11,073 INFO MainThread:2201339 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:41:11,085 INFO MainThread:2201339 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:41:11,091 INFO MainThread:2201339 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:41:12,371 INFO MainThread:2201339 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:41:12,437 INFO MainThread:2201339 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:41:12,437 INFO MainThread:2201339 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:41:12,778 INFO MainThread:2201339 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:41:12,779 INFO MainThread:2201339 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:41:12,780 INFO MainThread:2201339 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/you6ychq/tb, True +2025-01-07 17:41:20,638 INFO MainThread:2201339 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:45:24,466 WARNING MsgRouterThr:2201339 [router.py:message_loop():77] message_loop has been closed diff --git a/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb b/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a473ecbb3db0f5b48aec1133210c83eb9a5fecdf Binary files /dev/null and b/you6ychq/wandb/run-20250107_174111-you6ychq/run-you6ychq.wandb differ diff --git a/yyf8aq3v/episode_rosbags/aligned_depth_to_color_K.npy b/yyf8aq3v/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yyf8aq3v/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yyf8aq3v/episode_rosbags/cam_tf_world.npy b/yyf8aq3v/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/yyf8aq3v/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/yyf8aq3v/episode_rosbags/color_K.npy b/yyf8aq3v/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/yyf8aq3v/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/yyf8aq3v/episode_rosbags/depth_K.npy b/yyf8aq3v/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/yyf8aq3v/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/yyf8aq3v/eval_robot.log b/yyf8aq3v/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..217f85963a515e176aa24bf389d40a66702e7a77 --- /dev/null +++ b/yyf8aq3v/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 17:36:16,336][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 17:36:16,339][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/yyf8aq3v/wandb/debug-internal.log b/yyf8aq3v/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c68bc2ee9ea608b12f3d430cf6db9d1113f3e9e0 --- /dev/null +++ b/yyf8aq3v/wandb/debug-internal.log @@ -0,0 +1,752 @@ +2025-01-07 17:36:22,061 INFO StreamThr :2200445 [internal.py:wandb_internal():85] W&B internal server running at pid: 2200445, started at: 2025-01-07 17:36:22.061446 +2025-01-07 17:36:22,063 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status +2025-01-07 17:36:22,074 INFO WriterThread:2200445 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb +2025-01-07 17:36:22,075 DEBUG SenderThread:2200445 [sender.py:send():379] send: header +2025-01-07 17:36:22,082 DEBUG SenderThread:2200445 [sender.py:send():379] send: run +2025-01-07 17:36:22,320 INFO SenderThread:2200445 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files +2025-01-07 17:36:22,320 INFO SenderThread:2200445 [sender.py:_start_run_threads():1188] run started: yyf8aq3v with start time 1736289382.061578 +2025-01-07 17:36:22,336 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 17:36:22,336 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: check_version +2025-01-07 17:36:22,409 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 17:36:22,414 DEBUG HandlerThread:2200445 [system_info.py:__init__():26] System info init +2025-01-07 17:36:22,414 DEBUG HandlerThread:2200445 [system_info.py:__init__():41] System info init done +2025-01-07 17:36:22,414 INFO HandlerThread:2200445 [system_monitor.py:start():194] Starting system monitor +2025-01-07 17:36:22,414 INFO SystemMonitor:2200445 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 17:36:22,415 INFO HandlerThread:2200445 [system_monitor.py:probe():214] Collecting system info +2025-01-07 17:36:22,415 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 17:36:22,415 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started disk monitoring +2025-01-07 17:36:22,416 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 17:36:22,417 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started memory monitoring +2025-01-07 17:36:22,417 INFO SystemMonitor:2200445 [interfaces.py:start():188] Started network monitoring +2025-01-07 17:36:22,427 DEBUG HandlerThread:2200445 [system_info.py:probe():152] Probing system +2025-01-07 17:36:22,429 DEBUG HandlerThread:2200445 [system_info.py:_probe_git():137] Probing git +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:_probe_git():145] Probing git done +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:probe():200] Probing system done +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T22:36:22.427733', 'startedAt': '2025-01-07T22:36:22.049890', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.23825, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.272, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 763.361328125}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 17:36:22,433 INFO HandlerThread:2200445 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 17:36:22,433 INFO HandlerThread:2200445 [system_monitor.py:probe():227] Publishing system info +2025-01-07 17:36:22,433 DEBUG HandlerThread:2200445 [system_info.py:_save_code():44] Saving code +2025-01-07 17:36:22,439 DEBUG HandlerThread:2200445 [system_info.py:_save_code():67] Saving code done +2025-01-07 17:36:22,439 DEBUG HandlerThread:2200445 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 17:36:22,501 DEBUG HandlerThread:2200445 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 17:36:22,502 INFO HandlerThread:2200445 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 17:36:22,505 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 17:36:22,505 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 17:36:22,751 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/3jo5rt4f-diff.patch +2025-01-07 17:36:22,757 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/tgxibp13-wandb-metadata.json +2025-01-07 17:36:22,759 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/474yrsfp-code/FISH/eval_robot.py +2025-01-07 17:36:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 17:36:22,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: python_packages +2025-01-07 17:36:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:22,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:22,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:22,762 INFO HandlerThread:2200445 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/tb" + save: true +} +control { + relay_id: "b694256fd9024681b0cafe8a5a7a7ed6" +} +_info { + stream_id: "yyf8aq3v" +} + +2025-01-07 17:36:22,762 DEBUG HandlerThread:2200445 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 17:36:22,833 DEBUG SenderThread:2200445 [sender.py:send():379] send: telemetry +2025-01-07 17:36:22,833 DEBUG SenderThread:2200445 [sender.py:send():379] send: tbrecord +2025-01-07 17:36:22,834 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:36:22,834 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 with policy live +2025-01-07 17:36:23,025 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/3esqj776-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:36:23,321 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH +2025-01-07 17:36:23,322 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code +2025-01-07 17:36:23,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:24,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:25,321 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:25,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:26,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:27,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:27,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:28,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:29,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:30,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:31,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:32,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:33,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:33,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:34,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:35,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:36,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:37,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:37,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:37,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:38,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:38,911 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:39,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:40,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:41,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:42,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:43,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:44,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:44,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:45,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:46,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:47,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:48,115 DEBUG SenderThread:2200445 [sender.py:send():379] send: config +2025-01-07 17:36:48,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:49,327 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:49,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:50,165 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:50,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:51,328 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:51,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:52,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:36:52,759 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:36:52,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:53,328 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:53,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:54,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:55,329 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:55,556 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:36:55,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:56,329 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:36:56,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:57,330 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:36:57,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:58,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:36:59,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:00,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:00,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:01,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:02,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:03,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:04,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:05,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:06,257 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:06,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:07,332 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:07,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:07,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:07,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:08,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:09,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:10,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:11,333 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:11,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:12,243 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:12,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:13,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:14,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:15,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:16,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:17,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:17,766 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:18,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:19,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:20,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:21,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:22,417 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 17:37:22,418 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:37:22,759 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:22,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:22,805 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:22,892 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:23,767 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:24,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:25,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:26,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:27,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:28,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:28,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:29,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:30,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:31,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:32,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:33,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:33,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:34,769 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:35,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:36,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:36,941 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:37:36,942 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:37:36,942 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 with policy now +2025-01-07 17:37:36,942 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:37:36,943 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:37:36,943 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:37:37,162 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/kfxfaftv-media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos +2025-01-07 17:37:37,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:37:37,615 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/8oax5jws-media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:37:37,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:37,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:37,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:38,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:38,877 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:39,341 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:39,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:40,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:41,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:42,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:43,342 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:43,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:44,377 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:44,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:45,343 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:37:45,343 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:45,584 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/wzto8sho-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:37:45,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:46,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:47,344 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:47,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:48,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:49,779 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:49,837 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:50,779 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:51,345 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:37:51,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:52,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:37:52,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:37:52,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:37:52,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:53,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:54,780 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:54,856 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:37:55,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:56,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:57,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:58,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:37:59,781 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:00,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:00,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:01,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:02,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:03,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:04,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:05,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:05,782 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:06,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:07,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:07,760 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:07,805 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:08,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:09,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:10,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:10,886 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:11,783 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:12,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:13,353 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:13,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:14,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:15,784 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:16,252 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:16,628 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:38:16,631 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:38:16,631 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 with policy now +2025-01-07 17:38:16,632 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:16,632 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:16,632 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:38:16,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:16,851 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/u0w1hrt2-media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,354 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:38:17,354 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:17,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:38:17,629 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/fo2gg6ut-media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:38:17,788 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:18,785 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:19,355 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:19,794 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:20,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:21,356 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:21,405 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:21,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:22,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:38:22,764 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:22,764 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:22,809 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:23,786 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:24,786 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:25,357 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:25,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:26,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:27,178 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:27,788 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:28,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:29,184 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:29,184 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:29,184 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:38:29,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:38:29,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:38:29,787 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:30,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:31,359 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:31,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:32,312 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:32,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:33,360 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:33,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:34,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:35,361 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:38:35,789 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:36,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:37,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:37,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:37,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:37,872 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:38,790 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:39,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:40,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:41,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:42,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:43,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:43,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:44,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:45,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:46,791 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:47,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:48,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:49,765 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:49,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:50,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:51,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:52,419 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:38:52,760 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:38:52,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:38:52,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:53,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:54,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:54,876 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:55,792 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:56,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:57,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:58,793 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:59,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:38:59,877 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:38:59,917 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:38:59,917 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:38:59,918 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 with policy now +2025-01-07 17:38:59,918 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:38:59,918 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:38:59,918 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:00,138 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/fizdclx7-media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:39:00,641 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/67c7iyip-media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:39:00,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:01,369 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:01,797 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:02,802 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:03,798 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:04,801 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:05,371 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:05,685 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:05,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:06,799 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:07,371 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:07,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:07,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:07,809 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:08,015 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:08,015 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:08,015 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:08,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:08,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:08,619 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/uc69riqb-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:08,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:09,372 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:09,800 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:10,802 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:10,949 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:11,373 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:11,830 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:12,820 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:13,373 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:13,821 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:14,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:15,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:16,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:16,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:17,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:18,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:19,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:20,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:21,773 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:21,822 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:22,420 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:39:22,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:22,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:22,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:23,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:24,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:25,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:26,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:26,840 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:27,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:28,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:29,823 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:30,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:31,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:32,768 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:32,824 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:33,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:34,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:35,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:36,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:37,761 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:37,761 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:37,825 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:37,903 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:38,833 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:39,828 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:40,287 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:39:40,288 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 with policy now +2025-01-07 17:39:40,294 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:39:40,294 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:40,294 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:40,295 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:40,382 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:40,382 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:39:40,519 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/hzk97aqp-media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,794 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/1n1ct8aa-media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:39:40,831 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:41,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:41,835 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:42,383 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:42,831 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:43,770 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:43,836 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:44,839 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:45,384 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:45,835 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:46,873 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:47,841 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:48,773 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:48,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:49,385 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:49,840 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:50,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:39:50,836 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:51,070 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:39:51,070 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:39:51,070 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:39:51,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:39:51,386 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:51,847 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:52,420 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:39:52,763 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:39:52,763 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:39:52,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:53,387 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:53,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:53,869 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:54,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:55,388 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:55,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:56,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:57,389 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:39:57,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:58,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:39:59,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:39:59,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:00,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:01,842 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:02,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:03,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:04,771 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:04,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:05,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:06,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:07,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:07,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:07,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:08,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:09,843 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:09,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:10,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:11,844 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:12,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:13,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:14,845 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:15,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:15,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:16,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:17,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:18,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:19,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:20,772 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:20,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:21,846 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:22,421 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:40:22,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:22,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:22,847 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:23,849 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:24,143 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:24,143 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 with policy now +2025-01-07 17:40:24,148 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:24,149 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:24,149 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:24,149 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:24,367 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/tihhbsaj-media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval +2025-01-07 17:40:24,679 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/v30vykpu-media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:24,854 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:25,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:25,858 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:26,177 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:26,398 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:26,855 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:27,864 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:28,858 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:29,400 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:29,866 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:30,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:31,733 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:31,863 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:32,118 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:32,119 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:32,142 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:32,401 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:32,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:32,597 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:40:32,598 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:40:32,663 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/4z5hm4g1-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:32,839 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:32,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:33,047 INFO wandb-upload_2:2200445 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprjzsorss +2025-01-07 17:40:33,293 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:40:33,304 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:33,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media +2025-01-07 17:40:33,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval +2025-01-07 17:40:33,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table +2025-01-07 17:40:33,584 INFO SenderThread:2200445 [sender.py:send_request_log_artifact():1525] logged artifact run-yyf8aq3v-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk3Nzg0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxNTgyNw==', 'latestArtifact': None}} +2025-01-07 17:40:33,586 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:33,586 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 17:40:33,587 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:33,587 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,591 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,592 DEBUG SenderThread:2200445 [sender.py:send():379] send: telemetry +2025-01-07 17:40:33,592 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,598 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,610 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,620 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,620 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,771 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/b0zt5kaw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:33,773 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,773 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,792 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,792 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,798 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/kauhvnds-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:33,808 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,808 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,817 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,818 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,820 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,820 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,824 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,824 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,826 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,826 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,829 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,830 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:33,832 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:33,832 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:40:33,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:34,033 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/cisms8s4-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:34,223 INFO wandb-upload_1:2200445 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk4pfvn46 +2025-01-07 17:40:34,402 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:34,615 INFO SenderThread:2200445 [sender.py:send_request_log_artifact():1525] logged artifact run-yyf8aq3v-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTk3Nzk4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTcxNTkxOA==', 'latestArtifact': None}} +2025-01-07 17:40:34,615 DEBUG SenderThread:2200445 [sender.py:send():379] send: files +2025-01-07 17:40:34,615 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-07 17:40:34,615 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:34,616 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:34,616 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:34,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:34,888 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/pq63jbqt-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:35,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:35,403 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:35,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:36,860 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:37,617 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:37,762 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:40:37,762 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:40:37,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:38,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:39,861 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:40,877 DEBUG SenderThread:2200445 [sender.py:send():379] send: exit +2025-01-07 17:40:40,877 INFO SenderThread:2200445 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_exit():588] handling runtime: 258 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_exit():594] send defer +2025-01-07 17:40:40,878 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,878 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:40:40,878 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:40:40,878 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:40:40,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:40:40,879 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,879 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:40:40,879 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:40:40,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:40:40,879 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:40:40,880 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:40:40,880 DEBUG SystemMonitor:2200445 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:40:40,930 INFO HandlerThread:2200445 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send():379] send: stats +2025-01-07 17:40:40,931 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,931 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:40:40,931 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:40:40,931 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:40:40,932 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:40,932 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:40:41,405 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:43,406 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:40:43,406 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:47,129 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:40:47,130 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,130 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:40:47,130 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:40:47,130 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:47,131 DEBUG SenderThread:2200445 [sender.py:send():379] send: history +2025-01-07 17:40:47,131 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:40:47,131 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,131 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:47,132 DEBUG SenderThread:2200445 [sender.py:send():379] send: summary +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:40:47,132 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:40:47,132 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,133 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:40:47,133 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:40:47,133 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:47,133 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:40:47,133 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:47,133 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:40:47,408 INFO Thread-12 :2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:47,879 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:50,205 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:40:50,205 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:50,206 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,206 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:40:50,206 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,206 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:40:50,206 INFO SenderThread:2200445 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:40:50,208 INFO SenderThread:2200445 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:40:50,210 INFO SenderThread:2200445 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:40:50,224 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:40:50,225 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,225 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:40:50,225 DEBUG SenderThread:2200445 [sender.py:send():379] send: artifact +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 17:40:50,406 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:40:50,406 INFO SenderThread:2200445 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml config.yaml +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json wandb-summary.json +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt requirements.txt +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:40:50,409 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:50,410 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log output.log +2025-01-07 17:40:50,410 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch diff.patch +2025-01-07 17:40:50,413 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 +2025-01-07 17:40:50,414 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 17:40:50,415 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:40:50,415 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,415 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:40:50,416 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,416 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:40:50,416 INFO SenderThread:2200445 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:40:50,607 INFO wandb-upload_1:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml +2025-01-07 17:40:50,608 INFO wandb-upload_2:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json +2025-01-07 17:40:50,618 INFO wandb-upload_0:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt +2025-01-07 17:40:50,648 INFO wandb-upload_4:2200445 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log +2025-01-07 17:40:50,717 INFO wandb-upload_3:2200445 [upload_job.py:push():130] Uploaded file /tmp/tmptltu7panwandb/7ml17wwx-events.out.tfevents.1736289382.leonmkim-ROG-Strix-G15CS-G15CS.2200279.0 +2025-01-07 17:40:50,880 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:50,880 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:50,917 INFO Thread-11 (_thread_body):2200445 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:40:50,917 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,917 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:40:50,918 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,918 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:40:50,918 INFO SenderThread:2200445 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:40:50,919 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:50,919 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:40:50,919 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:40:50,919 INFO SenderThread:2200445 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:40:51,371 INFO SenderThread:2200445 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:40:51,371 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:40:51,371 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:51,371 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:40:51,371 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:40:51,372 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:40:51,372 INFO HandlerThread:2200445 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send():379] send: final +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send():379] send: footer +2025-01-07 17:40:51,372 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: defer +2025-01-07 17:40:51,372 INFO SenderThread:2200445 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:51,373 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:40:51,373 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:40:51,374 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:40:51,374 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:40:51,374 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:40:51,375 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:40:51,441 DEBUG SenderThread:2200445 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:40:51,442 INFO MainThread:2200445 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:40:51,443 DEBUG HandlerThread:2200445 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:40:51,443 INFO HandlerThread:2200445 [handler.py:finish():882] shutting down handler +2025-01-07 17:40:52,374 INFO WriterThread:2200445 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [sender.py:finish():1615] shutting down sender +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:40:52,442 INFO SenderThread:2200445 [file_pusher.py:join():175] waiting for file pusher diff --git a/yyf8aq3v/wandb/debug.log b/yyf8aq3v/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..11366aebea6dafed2874ae221fa2dc8ffdd26b99 --- /dev/null +++ b/yyf8aq3v/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Configure stats pid to 2200279 +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/settings +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug.log +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug-internal.log +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v'} +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():619] starting backend +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():623] setting up manager +2025-01-07 17:36:22,060 INFO MainThread:2200279 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:36:22,061 INFO MainThread:2200279 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:36:22,074 INFO MainThread:2200279 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:36:22,080 INFO MainThread:2200279 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:36:22,335 INFO MainThread:2200279 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:36:22,404 INFO MainThread:2200279 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:36:22,404 INFO MainThread:2200279 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:36:22,759 INFO MainThread:2200279 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:36:22,762 INFO MainThread:2200279 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:36:22,762 INFO MainThread:2200279 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/tb, True +2025-01-07 17:36:48,114 INFO MainThread:2200279 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:40:52,443 WARNING MsgRouterThr:2200279 [router.py:message_loop():77] message_loop has been closed diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98a9afe6b73903ad1d6b28eb27d5cf79ffbebdb5 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736289382 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1af31b2ff07685aba62f0764c3073e6e9867fe4d --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/diff.patch @@ -0,0 +1,21 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..6e694c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdff6fffa9b038190acdbc6f7b116097984799e --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 50, 0.5, 0.33669589463162186, 0.6620035767231229, 0.3672531694439306, 0.6438583284328754, 0.387978680454247, 0.6119417696967355, 0.4043694697511723, 0.5970221161648301]]} \ No newline at end of file diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..465c6f1dc5b30088270d1ccee09c25ac68c833d7 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90026e2343d0351af69a1f8a7c0e73e2229ceb186ccb6bfa557a8be5278e2132 +size 1148153 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef54867939d020f76e5b1dce3ac9678d5a724a3d --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:269892ed51d4f6e3e256c71742e308af88e94e3950f5cb2af75013a41e0809f4 +size 1152547 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8ef50361199ea6d8f40f8059e44ad93e54d4869 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/2_eval_2_651847148ddc37157f47.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651847148ddc37157f47c67bf89d6ec691eb37886c34e76041adfdf18b3e3aea +size 1130336 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..050aab2ce2b5bd36e02f36568b80eb07481ba3fa --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e672402e025995fe93eba5820cc297d120d4275184b78a82d6ad559304d292f +size 1144841 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8b54a0f7dec6c9aff2749d2c4c48961175442fa7 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1fe43311f51687c2c7a14e0e7261f20b724a3e9c42ef4004f5d9f7073493a6f +size 1223808 diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ec7c27049c5336e5b686b7895ecb49fdbde2b044 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736289408.115077]: resetting environment +[INFO] [1736289408.118344]: cleared current plan +[INFO] [1736289408.119594]: moving to home +[INFO] [1736289409.121283]: reached home +[INFO] [1736289409.122662]: reset action history +[INFO] [1736289411.830444]: environment reset +Starting episode 0 +[INFO] [1736289411.833728]: resetting environment +[INFO] [1736289411.838725]: cleared current plan +[INFO] [1736289411.839814]: moving to home +[INFO] [1736289412.842326]: reached home +[INFO] [1736289412.843943]: reset action history +[INFO] [1736289415.552382]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736289425.250469]: resetting environment +[INFO] [1736289425.253636]: cleared current plan +[INFO] [1736289425.254475]: moving to home +[INFO] [1736289426.256823]: reached home +[INFO] [1736289426.258777]: reset action history +[INFO] [1736289428.965803]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/episode_rosbags/episode_0_2025-01-07-17-37-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2200279.log +started roslaunch server http://158.130.50.37:33561/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2200561] +process[print_text-2]: started with pid [2200562] +process[rosbag_record-3]: started with pid [2200586] +started bagging! +[INFO] [1736289455.949302]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736289456.704528]: Stored episode 1. +[INFO] [1736289456.705575]: Saving video... +[INFO] [1736289456.898947]: Video saved! +Starting episode 1 +[INFO] [1736289456.915664]: resetting environment +[INFO] [1736289456.920536]: cleared current plan +[INFO] [1736289456.920793]: moving to home +[INFO] [1736289460.623518]: reached home +[INFO] [1736289460.624996]: reset action history +[INFO] [1736289463.336231]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289464.838144]: resetting environment +[INFO] [1736289464.841538]: cleared current plan +[INFO] [1736289464.841737]: moving to home +[INFO] [1736289465.843255]: reached home +[INFO] [1736289465.843656]: reset action history +[INFO] [1736289468.600189]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/episode_rosbags/episode_1_2025-01-07-17-37-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2200279.log +started roslaunch server http://158.130.50.37:37415/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2200707] +process[print_text-5]: started with pid [2200708] +process[rosbag_record-6]: started with pid [2200709] +started bagging! +For topic gripper_width: timestamp difference is -207439954 for nearest: 1736289491678830331 - target: 1736289491886270285 at idx 249 +[INFO] [1736289495.535756]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736289496.250219]: Stored episode 2. +[INFO] [1736289496.252537]: Saving video... +[INFO] [1736289496.590795]: Video saved! +Starting episode 2 +[INFO] [1736289496.632951]: resetting environment +[INFO] [1736289496.653575]: cleared current plan +[INFO] [1736289496.654871]: moving to home +[INFO] [1736289500.361325]: reached home +[INFO] [1736289500.363019]: reset action history +[INFO] [1736289503.131219]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289508.593609]: resetting environment +[INFO] [1736289508.598645]: cleared current plan +[INFO] [1736289508.601475]: moving to home +[INFO] [1736289509.604129]: reached home +[INFO] [1736289509.604581]: reset action history +[INFO] [1736289512.312315]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/episode_rosbags/episode_2_2025-01-07-17-38-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2200279.log +started roslaunch server http://158.130.50.37:39083/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2200842] +process[print_text-8]: started with pid [2200843] +process[rosbag_record-9]: started with pid [2200867] +started bagging! +[INFO] [1736289539.252023]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736289539.663426]: Stored episode 3. +[INFO] [1736289539.666364]: Saving video... +[INFO] [1736289539.876808]: Video saved! +Starting episode 3 +[INFO] [1736289539.909740]: resetting environment +[INFO] [1736289539.939353]: cleared current plan +[INFO] [1736289539.939845]: moving to home +[INFO] [1736289543.644032]: reached home +[INFO] [1736289543.650284]: reset action history +[INFO] [1736289546.490262]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289547.878055]: resetting environment +[INFO] [1736289547.895366]: cleared current plan +[INFO] [1736289547.896807]: moving to home +[INFO] [1736289548.901095]: reached home +[INFO] [1736289548.905899]: reset action history +[INFO] [1736289551.632557]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/episode_rosbags/episode_3_2025-01-07-17-39-11.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2200279.log +started roslaunch server http://158.130.50.37:34309/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2200977] +process[print_text-11]: started with pid [2200978] +process[rosbag_record-12]: started with pid [2201002] +started bagging! +[INFO] [1736289578.792344]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1736289579.601169]: Stored episode 4. +[INFO] [1736289579.605133]: Saving video... +[INFO] [1736289580.239522]: Video saved! +Starting episode 4 +[INFO] [1736289580.305261]: resetting environment +[INFO] [1736289580.314563]: cleared current plan +[INFO] [1736289580.324068]: moving to home +[WARN] [1736289580.140280]: Plan exhausted +[WARN] [1736289580.197186]: Plan exhausted +[WARN] [1736289580.221803]: Plan exhausted +[WARN] [1736289580.261941]: Plan exhausted +[WARN] [1736289580.300811]: Plan exhausted +[INFO] [1736289584.034577]: reached home +[INFO] [1736289584.038066]: reset action history +[INFO] [1736289586.831069]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736289590.268064]: resetting environment +[INFO] [1736289590.291523]: cleared current plan +[INFO] [1736289590.292944]: moving to home +[INFO] [1736289591.299975]: reached home +[INFO] [1736289591.316544]: reset action history +[INFO] [1736289594.095484]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/episode_rosbags/episode_4_2025-01-07-17-39-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2200279.log +started roslaunch server http://158.130.50.37:42507/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2201148] +process[print_text-14]: started with pid [2201149] +process[rosbag_record-15]: started with pid [2201150] +started bagging! +[INFO] [1736289622.769467]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736289623.557882]: Stored episode 5. +[INFO] [1736289623.558627]: Saving video... +[INFO] [1736289624.083335]: Video saved! +[INFO] [1736289624.151431]: resetting environment +[INFO] [1736289624.170773]: cleared current plan +[INFO] [1736289624.176978]: moving to home +[WARN] [1736289624.100265]: Plan exhausted +[WARN] [1736289624.142659]: Plan exhausted +[INFO] [1736289627.882634]: reached home +[INFO] [1736289627.888146]: reset action history +[INFO] [1736289630.692858]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b6ade9f5926f767b9dbe003cdfe2983a34878ba7 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T22:36:22.427733", + "startedAt": "2025-01-07T22:36:22.049890", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.23825, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.272, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.943, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 763.361328125 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dfb5f5c07e0cc97143dee3b679a6c571901b1c47 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "90026e2343d0351af69a1f8a7c0e73e2229ceb186ccb6bfa557a8be5278e2132", "size": 1148153, "path": "media/videos/eval/0_eval_0_90026e2343d0351af69a.mp4"}, "global_step": 664, "_timestamp": 1736289633.1792905, "_runtime": 251.11771249771118, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "269892ed51d4f6e3e256c71742e308af88e94e3950f5cb2af75013a41e0809f4", "size": 1152547, "path": "media/videos/eval/1_eval_1_269892ed51d4f6e3e256.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "651847148ddc37157f47c67bf89d6ec691eb37886c34e76041adfdf18b3e3aea", "size": 1130336, "path": "media/videos/eval/2_eval_2_651847148ddc37157f47.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5e672402e025995fe93eba5820cc297d120d4275184b78a82d6ad559304d292f", "size": 1144841, "path": "media/videos/eval/3_eval_3_5e672402e025995fe93e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "d1fe43311f51687c2c7a14e0e7261f20b724a3e9c42ef4004f5d9f7073493a6f", "size": 1223808, "path": "media/videos/eval/4_eval_4_d1fe43311f51687c2c7a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://vp4n4r5naoz28gumff9e4ivs1ym2fvix8chmt3lfcszl8zlsucos1gcl2y02ozbm77bq0xgniya9kekhmfp1blu63lb41yjy0ptvo7ww7apdq855wu1th08ay4yzqxxp/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tpze26c4vlwpz61pdilznbjgki7i6ycaa7cru9fzyvmw7vr96ge4zvpjrq4usukaxdnwus2z80uw784oejyqc412fqrk4vrjnkul0qrwrmo18oc74pa1x8svs3p9nj46:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 50, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.33669589463162186, "total_success_rate_ub_0.01": 0.6620035767231229, "total_success_rate_lb_0.025": 0.3672531694439306, "total_success_rate_ub_0.025": 0.6438583284328754, "total_success_rate_lb_0.05": 0.387978680454247, "total_success_rate_ub_0.05": 0.6119417696967355, "total_success_rate_lb_0.1": 0.4043694697511723, "total_success_rate_ub_0.1": 0.5970221161648301, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2f3f136f2937a5b3202d8da849c2ef52e3ba1205b3daba868e9420950bc47aef", "size": 505, "artifact_path": "wandb-client-artifact://0xt73k9pnips5d51w20hqhhn0k7gyiogy87uev4labrwjtuwuscez455mhuevl0ac3mo26b2tkn0ln7c6r76kehkt1ytuuedbc9xheqvxnrg5gas3stds08jvbgnpha2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://0b70cbzod6so7wswdqbqafigkl8srodjuu1uiqs97jyz4lcwi853u4wuzimwpech52z5vvxcf443ztgqfgpufuk50mg84qmxen0wkqz2q23k39dxwpqm9aibcecayrai:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 258}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.33669590950012207, "eval/total_success_rate_ub_0.01": 0.6620035767555237, "eval/total_success_rate_lb_0.025": 0.3672531843185425, "eval/total_success_rate_ub_0.025": 0.6438583135604858, "eval/total_success_rate_lb_0.05": 0.3879786729812622, "eval/total_success_rate_ub_0.05": 0.6119417548179626, "eval/total_success_rate_lb_0.1": 0.4043694734573364, "eval/total_success_rate_ub_0.1": 0.5970221161842346} \ No newline at end of file diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug.log b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..11366aebea6dafed2874ae221fa2dc8ffdd26b99 --- /dev/null +++ b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Configure stats pid to 2200279 +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/settings +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug.log +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/logs/debug-internal.log +2025-01-07 17:36:22,052 INFO MainThread:2200279 [wandb_init.py:init():569] calling init triggers +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v'} +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():619] starting backend +2025-01-07 17:36:22,053 INFO MainThread:2200279 [wandb_init.py:init():623] setting up manager +2025-01-07 17:36:22,060 INFO MainThread:2200279 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 17:36:22,061 INFO MainThread:2200279 [wandb_init.py:init():631] backend started and connected +2025-01-07 17:36:22,074 INFO MainThread:2200279 [wandb_init.py:init():720] updated telemetry +2025-01-07 17:36:22,080 INFO MainThread:2200279 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 17:36:22,335 INFO MainThread:2200279 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 17:36:22,404 INFO MainThread:2200279 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 17:36:22,404 INFO MainThread:2200279 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 17:36:22,759 INFO MainThread:2200279 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 17:36:22,760 INFO MainThread:2200279 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 17:36:22,762 INFO MainThread:2200279 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 17:36:22,762 INFO MainThread:2200279 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/yyf8aq3v/tb, True +2025-01-07 17:36:48,114 INFO MainThread:2200279 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 17:40:52,443 WARNING MsgRouterThr:2200279 [router.py:message_loop():77] message_loop has been closed diff --git a/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5ad845ee4a17c608309cb8684c77de42812a3d92 Binary files /dev/null and b/yyf8aq3v/wandb/run-20250107_173622-yyf8aq3v/run-yyf8aq3v.wandb differ