diff --git a/.gitattributes b/.gitattributes index ab1ef8c68f28e00b11d578ecd5d6e91083acd376..c9dc8a7af9a646810742f45635340ac66e9eecf2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -288,3 +288,88 @@ kccehv5k/episode_rosbags/episode_1_2024-12-31-20-48-15.bag filter=lfs diff=lfs m opf0nkwg/episode_rosbags/episode_4_2025-01-07-22-32-48.bag filter=lfs diff=lfs merge=lfs -text jqiikotd/episode_rosbags/episode_0_2025-01-07-17-23-40.bag filter=lfs diff=lfs merge=lfs -text fnq3cpdm/episode_rosbags/episode_3_2025-01-06-17-16-42.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_8_2025-01-07-22-35-50.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_17_2025-01-07-22-43-24.bag filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/episode_rosbags/episode_1_2025-01-06-17-14-46.bag filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/episode_rosbags/episode_4_2025-01-06-17-17-26.bag filter=lfs diff=lfs merge=lfs -text +kccehv5k/episode_rosbags/episode_0_2024-12-31-20-47-35.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_0_2025-01-07-22-29-47.bag filter=lfs diff=lfs merge=lfs -text +fnq3cpdm/episode_rosbags/episode_0_2025-01-06-17-13-55.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_13_2025-01-07-22-39-36.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_7_2025-01-07-22-35-02.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb filter=lfs diff=lfs merge=lfs -text +hle5fmwc/episode_rosbags/episode_1_2025-01-06-20-56-03.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/episode_rosbags/episode_3_2025-01-06-20-57-23.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/episode_rosbags/episode_0_2025-01-06-20-55-22.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_15_2025-01-07-22-41-37.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_19_2025-01-07-22-45-14.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +hle5fmwc/episode_rosbags/episode_2_2025-01-06-20-56-44.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_10_2025-01-07-22-37-20.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_6_2025-01-07-22-34-18.bag filter=lfs diff=lfs merge=lfs -text +hle5fmwc/episode_rosbags/episode_4_2025-01-06-20-58-20.bag filter=lfs diff=lfs merge=lfs -text +c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +c9fefav0/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_1_2025-01-07-22-30-39.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_14_2025-01-07-22-40-45.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_18_2025-01-07-22-44-20.bag filter=lfs diff=lfs merge=lfs -text +rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_9_2025-01-07-22-36-34.bag filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_3_2025-01-07-22-32-04.bag filter=lfs diff=lfs merge=lfs -text +rgxendfg/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +rgxendfg/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +opf0nkwg/episode_rosbags/episode_16_2025-01-07-22-42-29.bag filter=lfs diff=lfs merge=lfs -text diff --git a/c9fefav0/.hydra/config.yaml b/c9fefav0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/c9fefav0/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/c9fefav0/.hydra/hydra.yaml b/c9fefav0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81e433c98eb193d66c2b5bbc13d44f9504cf47fb --- /dev/null +++ b/c9fefav0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/c9fefav0/.hydra/overrides.yaml b/c9fefav0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/c9fefav0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/c9fefav0/episode_rosbags/aligned_depth_to_color_K.npy b/c9fefav0/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/c9fefav0/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/c9fefav0/episode_rosbags/cam_tf_world.npy b/c9fefav0/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/c9fefav0/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/c9fefav0/episode_rosbags/color_K.npy b/c9fefav0/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/c9fefav0/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/c9fefav0/episode_rosbags/depth_K.npy b/c9fefav0/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/c9fefav0/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/c9fefav0/eval_robot.log b/c9fefav0/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..60718d7fcb77b2340f66b8b4b3d3e2bba8a2701d --- /dev/null +++ b/c9fefav0/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 21:05:54,113][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 21:05:54,116][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/c9fefav0/eval_video/0_eval.mp4 b/c9fefav0/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf5d78579c18e4795530351a8cb5ab682f215f4a --- /dev/null +++ b/c9fefav0/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6960c47621db1ed25f24b27e4c1ac777c82b6889a68ffd14ab674a6733b385e1 +size 1134395 diff --git a/c9fefav0/eval_video/1_eval.mp4 b/c9fefav0/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..06c84ded143ecc71916e1937e29e3f1cd8b40638 --- /dev/null +++ b/c9fefav0/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e2b173de6e4d2512f5dd083c82685dd82fbb0ca632f6345727468c5f48ccac +size 1192512 diff --git a/c9fefav0/eval_video/2_eval.mp4 b/c9fefav0/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6eac640d5a8bec7b366550b2d7a4dde2a3286736 --- /dev/null +++ b/c9fefav0/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf7d1cec72ac69f865682a674836bcec8d12414f0914f8c0a5d7a662b9f06f7 +size 1130234 diff --git a/c9fefav0/eval_video/3_eval.mp4 b/c9fefav0/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09fb07e4237050af825ed1b01268e939bb70bd79 --- /dev/null +++ b/c9fefav0/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2898cfd9388e3dddd2d6aae0bcde25d9b76ff152c17d68c983c8f9d0ffb2c87a +size 1133700 diff --git a/c9fefav0/eval_video/4_eval.mp4 b/c9fefav0/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..304565924e47122dd7c910ff5fae456f47c78c2b --- /dev/null +++ b/c9fefav0/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:635826221e5f85ee234a0ccd9dbcd1e1bf368c7e54ca2949f80ca6b5f9a8b145 +size 1213859 diff --git a/c9fefav0/tb/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 b/c9fefav0/tb/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 new file mode 100644 index 0000000000000000000000000000000000000000..3f370df260b3ea7461cf87050c5cf437a75037a3 --- /dev/null +++ b/c9fefav0/tb/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0465e2a7134e1f173643824bae4308c31ccc595ab02ab67deee6e57e0de95791 +size 6539 diff --git a/c9fefav0/wandb/debug-internal.log b/c9fefav0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..657276e33e3895c81132f93eb1b0622aea0d3c33 --- /dev/null +++ b/c9fefav0/wandb/debug-internal.log @@ -0,0 +1,750 @@ +2025-01-06 21:05:59,930 INFO StreamThr :2118961 [internal.py:wandb_internal():85] W&B internal server running at pid: 2118961, started at: 2025-01-06 21:05:59.930454 +2025-01-06 21:05:59,931 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status +2025-01-06 21:05:59,944 INFO WriterThread:2118961 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb +2025-01-06 21:05:59,944 DEBUG SenderThread:2118961 [sender.py:send():379] send: header +2025-01-06 21:05:59,952 DEBUG SenderThread:2118961 [sender.py:send():379] send: run +2025-01-06 21:06:00,233 INFO SenderThread:2118961 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files +2025-01-06 21:06:00,233 INFO SenderThread:2118961 [sender.py:_start_run_threads():1188] run started: c9fefav0 with start time 1736215559.931128 +2025-01-06 21:06:00,254 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 21:06:00,254 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: check_version +2025-01-06 21:06:00,405 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 21:06:00,410 DEBUG HandlerThread:2118961 [system_info.py:__init__():26] System info init +2025-01-06 21:06:00,410 DEBUG HandlerThread:2118961 [system_info.py:__init__():41] System info init done +2025-01-06 21:06:00,410 INFO HandlerThread:2118961 [system_monitor.py:start():194] Starting system monitor +2025-01-06 21:06:00,410 INFO SystemMonitor:2118961 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 21:06:00,410 INFO HandlerThread:2118961 [system_monitor.py:probe():214] Collecting system info +2025-01-06 21:06:00,411 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 21:06:00,411 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started disk monitoring +2025-01-06 21:06:00,412 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 21:06:00,412 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started memory monitoring +2025-01-06 21:06:00,413 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started network monitoring +2025-01-06 21:06:00,423 DEBUG HandlerThread:2118961 [system_info.py:probe():152] Probing system +2025-01-06 21:06:00,424 DEBUG HandlerThread:2118961 [system_info.py:_probe_git():137] Probing git +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:_probe_git():145] Probing git done +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:probe():200] Probing system done +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T02:06:00.423214', 'startedAt': '2025-01-07T02:05:59.919242', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.779375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 545.061149597168}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 21:06:00,429 INFO HandlerThread:2118961 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 21:06:00,429 INFO HandlerThread:2118961 [system_monitor.py:probe():227] Publishing system info +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:_save_code():44] Saving code +2025-01-06 21:06:00,431 DEBUG HandlerThread:2118961 [system_info.py:_save_code():67] Saving code done +2025-01-06 21:06:00,431 DEBUG HandlerThread:2118961 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 21:06:00,493 DEBUG HandlerThread:2118961 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 21:06:00,494 INFO HandlerThread:2118961 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 21:06:00,497 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:00,497 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 21:06:00,497 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 21:06:00,498 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 21:06:00,722 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/d02znp1c-wandb-metadata.json +2025-01-06 21:06:00,726 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 21:06:00,726 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: python_packages +2025-01-06 21:06:00,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:00,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:00,729 INFO HandlerThread:2118961 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/tb" + save: true +} +control { + relay_id: "b39b9ea63fe7412f80c74c56999bbc9f" +} +_info { + stream_id: "c9fefav0" +} + +2025-01-06 21:06:00,730 DEBUG HandlerThread:2118961 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 21:06:00,759 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/0kw4w1kr-code/FISH/eval_robot.py +2025-01-06 21:06:00,775 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/5zpoosz0-diff.patch +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: telemetry +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: tbrecord +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:00,832 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 with policy live +2025-01-06 21:06:01,011 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/1tzoylhq-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt +2025-01-06 21:06:01,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH +2025-01-06 21:06:01,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code +2025-01-06 21:06:01,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:02,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:03,234 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:03,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:04,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:05,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:05,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:06,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:07,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:08,698 DEBUG SenderThread:2118961 [sender.py:send():379] send: config +2025-01-06 21:06:08,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:09,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:10,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:11,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:11,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:11,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:12,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:13,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:14,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:15,237 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:15,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:15,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:16,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:17,177 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:17,238 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:17,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:18,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:19,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:20,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:21,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:22,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:22,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:23,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:24,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:25,240 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:25,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:26,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:27,241 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:27,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:28,393 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:28,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:29,241 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:29,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:30,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:30,727 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:30,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:31,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:32,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:33,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:33,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:34,243 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:06:34,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:35,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:36,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:37,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:38,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:39,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:39,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:40,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:41,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:42,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:43,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:44,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:45,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:45,727 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:45,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:45,779 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:46,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:47,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:48,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:49,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:50,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:51,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:51,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:52,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:53,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:54,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:55,053 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:55,053 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 with policy now +2025-01-06 21:06:55,054 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:06:55,055 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:06:55,055 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:06:55,055 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:06:55,248 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media +2025-01-06 21:06:55,420 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/r23wablw-media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:55,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:55,759 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/0u9q9fb7-media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:56,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:56,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:57,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:57,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:58,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:59,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:00,413 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 21:07:00,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:07:00,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:00,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:01,250 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:01,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:02,221 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:02,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:03,251 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:03,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:04,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:05,252 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:05,484 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mg199rqz-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:05,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:06,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:07,253 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:07,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:08,041 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:08,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:09,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:10,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:11,254 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:11,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:12,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:13,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:13,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:14,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:15,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:15,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:16,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:17,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:18,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:18,838 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:19,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:20,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:21,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:22,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:23,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:24,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:24,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:25,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:26,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:27,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:28,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:29,260 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:29,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:30,345 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:30,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:07:30,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:30,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:30,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:31,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:32,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:33,261 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:33,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:34,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:35,682 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:35,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:36,747 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:36,969 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:07:36,969 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:07:36,969 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 with policy now +2025-01-06 21:07:36,970 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:07:36,970 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:07:36,971 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:07:37,262 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:07:37,263 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/3l9ox851-media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,558 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/8371k7ey-media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:38,750 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:39,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:39,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:40,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:40,758 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:41,755 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:42,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:43,264 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:43,748 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:44,748 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:45,265 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:45,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:45,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:45,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:45,876 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:46,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:47,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:48,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:49,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:50,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:51,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:51,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:52,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:53,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:54,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:55,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:55,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:56,140 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:07:56,141 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:07:56,141 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:07:56,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:07:56,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:56,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:57,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:57,333 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:57,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:58,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:59,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:00,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:08:00,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:00,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:01,271 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:01,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:02,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:03,061 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:03,271 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:03,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:04,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:05,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:06,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:07,273 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:07,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:08,601 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:08,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:09,274 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:09,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:10,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:11,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:12,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:13,275 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:13,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:14,057 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:14,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:15,276 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:15,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:15,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:16,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:17,276 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:17,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:18,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:19,277 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:19,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:20,042 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:20,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:21,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:22,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:23,278 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:23,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:24,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:25,241 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:25,279 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:25,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:26,774 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:27,169 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:08:27,170 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 with policy now +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:08:27,170 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:08:27,472 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/71vu78cf-media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,769 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/vebbpivl-media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:28,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:29,281 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:29,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:30,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:08:30,415 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:30,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:30,733 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:30,772 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:31,780 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:32,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:33,282 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:33,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:34,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:35,283 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:35,493 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:35,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:36,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:37,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:37,993 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:08:37,993 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:08:37,994 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:08:38,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:08:38,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:08:38,528 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/i4f3rr73-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:08:38,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:39,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:39,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:40,673 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:40,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:41,285 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:41,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:42,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:43,286 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:43,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:44,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:45,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:45,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:45,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:45,792 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:46,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:47,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:48,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:49,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:50,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:51,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:51,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:52,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:53,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:54,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:55,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:56,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:56,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:57,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:58,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:59,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:00,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:09:00,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:00,729 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:00,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:01,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:01,844 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:02,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:03,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:04,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:05,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:06,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:07,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:07,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:09,294 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:09,798 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:09,803 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:09:09,803 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 with policy now +2025-01-06 21:09:09,845 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:09:09,846 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:09,846 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:09,846 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:10,068 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/2k9al4yi-media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:09:10,591 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/m6cjje27-media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,792 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:11,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:11,791 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:12,743 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:12,794 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:13,793 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:14,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:15,296 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:15,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:15,741 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:15,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:16,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:17,297 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:17,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:18,093 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:18,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:19,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:20,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:21,802 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:22,810 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:23,047 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:23,047 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:23,047 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:09:23,645 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:23,818 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:24,816 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:25,300 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:25,811 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:26,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:27,301 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:27,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:28,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:28,963 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:29,301 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:29,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:30,416 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:09:30,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:30,732 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:30,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:31,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:32,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:33,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:34,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:34,825 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:35,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:36,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:37,305 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:37,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:38,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:39,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:40,744 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:40,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:41,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:42,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:43,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:44,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:45,308 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:45,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:45,732 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:45,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:45,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:46,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:47,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:48,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:49,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:50,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:51,744 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:51,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:52,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:53,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:54,823 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:54,875 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:09:54,875 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 with policy now +2025-01-06 21:09:54,876 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:09:54,876 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:54,876 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:54,877 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:55,136 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/s6cpd9mv-media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:09:55,620 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/7somprq8-media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,823 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:56,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:56,825 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:57,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:57,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:58,827 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:59,313 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:59,846 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:00,416 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:10:00,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:10:00,736 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:10:00,832 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:01,848 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:02,221 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:02,839 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:03,315 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:03,315 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:03,557 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mzr8ew5q-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:03,568 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:10:03,569 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:10:03,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:10:03,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:04,026 INFO wandb-upload_0:2118961 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpil_x2tbd +2025-01-06 21:10:04,199 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:10:04,206 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media +2025-01-06 21:10:04,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval +2025-01-06 21:10:04,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table +2025-01-06 21:10:04,413 INFO SenderThread:2118961 [sender.py:send_request_log_artifact():1525] logged artifact run-c9fefav0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM4MjE4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2OTMzNw==', 'latestArtifact': None}} +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:10:04,414 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,415 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: telemetry +2025-01-06 21:10:04,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:04,416 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,419 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,419 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,420 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,420 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,421 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,421 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,422 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,422 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,423 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,423 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,423 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,424 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,424 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,424 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,424 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,425 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,425 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,425 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,425 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,426 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,426 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,426 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,426 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,427 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:10:04,618 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mxdnn4xs-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:04,619 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/36bzv95d-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:04,804 INFO wandb-upload_0:2118961 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6ue9dgnr +2025-01-06 21:10:04,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:04,861 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/kf1v0fpa-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:05,236 INFO SenderThread:2118961 [sender.py:send_request_log_artifact():1525] logged artifact run-c9fefav0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM4MjIxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2OTM0NA==', 'latestArtifact': None}} +2025-01-06 21:10:05,236 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:10:05,236 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json with policy now +2025-01-06 21:10:05,237 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:05,237 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:05,237 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:05,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:05,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:05,476 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/vzfhfdm4-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:05,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:06,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:07,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:07,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:08,786 DEBUG SenderThread:2118961 [sender.py:send():379] send: exit +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():588] handling runtime: 248 +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():594] send defer +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 21:10:08,787 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 1 +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 21:10:08,787 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 2 +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 21:10:08,788 INFO HandlerThread:2118961 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 21:10:08,788 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 21:10:08,788 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 21:10:08,789 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 21:10:08,789 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 21:10:08,838 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 21:10:08,839 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 21:10:08,839 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined network monitor +2025-01-06 21:10:08,839 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,839 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 21:10:08,839 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 3 +2025-01-06 21:10:08,839 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:10:08,840 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,840 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 21:10:08,840 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,840 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 21:10:08,840 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 4 +2025-01-06 21:10:08,840 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,840 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 21:10:09,318 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:11,319 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,108 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 21:10:15,108 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 5 +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:15,109 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,110 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:15,110 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 21:10:15,110 DEBUG SenderThread:2118961 [sender.py:send():379] send: summary +2025-01-06 21:10:15,113 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:15,114 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 6 +2025-01-06 21:10:15,114 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,114 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 21:10:15,114 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 21:10:15,117 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:15,250 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 7 +2025-01-06 21:10:15,250 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,250 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 21:10:15,251 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,251 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 21:10:15,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:15,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:10:15,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:17,280 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 8 +2025-01-06 21:10:17,281 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:17,281 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:17,281 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 21:10:17,281 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:17,281 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 21:10:17,281 INFO SenderThread:2118961 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 21:10:17,285 INFO SenderThread:2118961 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 21:10:17,287 INFO SenderThread:2118961 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 21:10:17,299 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 9 +2025-01-06 21:10:17,299 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:17,299 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 21:10:17,300 DEBUG SenderThread:2118961 [sender.py:send():379] send: artifact +2025-01-06 21:10:17,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 21:10:17,506 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 21:10:17,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,322 INFO SenderThread:2118961 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml config.yaml +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json wandb-summary.json +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:18,325 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt requirements.txt +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json wandb-metadata.json +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log output.log +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch diff.patch +2025-01-06 21:10:18,327 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 10 +2025-01-06 21:10:18,332 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,332 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,332 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 21:10:18,332 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:10:18,507 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:10:18,526 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:18,532 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:18,549 INFO wandb-upload_3:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/tyb1g2h5-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:18,561 INFO wandb-upload_4:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt +2025-01-06 21:10:18,762 INFO Thread-11 (_thread_body):2118961 [sender.py:transition_state():614] send defer: 11 +2025-01-06 21:10:18,762 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,762 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 21:10:18,762 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,762 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 21:10:18,762 INFO SenderThread:2118961 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 21:10:18,763 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 12 +2025-01-06 21:10:18,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,764 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 21:10:18,764 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,764 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 21:10:18,764 INFO SenderThread:2118961 [file_stream.py:finish():601] file stream finish called +2025-01-06 21:10:18,788 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,934 INFO SenderThread:2118961 [file_stream.py:finish():605] file stream finish is done +2025-01-06 21:10:18,934 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 13 +2025-01-06 21:10:18,934 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,934 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,935 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 14 +2025-01-06 21:10:18,935 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,935 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send():379] send: final +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send():379] send: footer +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,937 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 21:10:18,937 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,937 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 21:10:18,937 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: server_info +2025-01-06 21:10:18,938 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:18,992 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 21:10:18,994 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 21:10:18,994 INFO HandlerThread:2118961 [handler.py:finish():882] shutting down handler +2025-01-06 21:10:19,937 INFO WriterThread:2118961 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [sender.py:finish():1615] shutting down sender +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [file_pusher.py:join():175] waiting for file pusher diff --git a/c9fefav0/wandb/debug.log b/c9fefav0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a2c47ccff05cdc8e6a86b287a2a47909864710ea --- /dev/null +++ b/c9fefav0/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Configure stats pid to 2118796 +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/settings +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug.log +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug-internal.log +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():569] calling init triggers +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0'} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():619] starting backend +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():623] setting up manager +2025-01-06 21:05:59,929 INFO MainThread:2118796 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 21:05:59,930 INFO MainThread:2118796 [wandb_init.py:init():631] backend started and connected +2025-01-06 21:05:59,943 INFO MainThread:2118796 [wandb_init.py:init():720] updated telemetry +2025-01-06 21:05:59,950 INFO MainThread:2118796 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 21:06:00,254 INFO MainThread:2118796 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 21:06:00,400 INFO MainThread:2118796 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 21:06:00,400 INFO MainThread:2118796 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 21:06:00,728 INFO MainThread:2118796 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 21:06:00,729 INFO MainThread:2118796 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 21:06:00,729 INFO MainThread:2118796 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/tb, True +2025-01-06 21:06:08,698 INFO MainThread:2118796 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 21:10:19,994 WARNING MsgRouterThr:2118796 [router.py:message_loop():77] message_loop has been closed diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97fbd33cf9d6ae0519adea65bf91825786d0e06d --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736215559 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e24d60575094ab086c08b663f56c41b1223113ec --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch @@ -0,0 +1,65 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..3e5ed57 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,12 +133,19 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9239b8930ef50a5961d10da1c1fa840ac6307567 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 35, 0.5714285714285714, 0.3716529608029522, 0.7538510262458453, 0.410685271042567, 0.7356967329507549, 0.4360231012233896, 0.6986087858279691, 0.4541952274831351, 0.6842993311215397]]} \ No newline at end of file diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cf5d78579c18e4795530351a8cb5ab682f215f4a --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6960c47621db1ed25f24b27e4c1ac777c82b6889a68ffd14ab674a6733b385e1 +size 1134395 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..06c84ded143ecc71916e1937e29e3f1cd8b40638 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e2b173de6e4d2512f5dd083c82685dd82fbb0ca632f6345727468c5f48ccac +size 1192512 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6eac640d5a8bec7b366550b2d7a4dde2a3286736 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf7d1cec72ac69f865682a674836bcec8d12414f0914f8c0a5d7a662b9f06f7 +size 1130234 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..09fb07e4237050af825ed1b01268e939bb70bd79 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2898cfd9388e3dddd2d6aae0bcde25d9b76ff152c17d68c983c8f9d0ffb2c87a +size 1133700 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..304565924e47122dd7c910ff5fae456f47c78c2b --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:635826221e5f85ee234a0ccd9dbcd1e1bf368c7e54ca2949f80ca6b5f9a8b145 +size 1213859 diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0ce59d5d68e452d81193841bb3ab55e26686836b --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736215568.698674]: resetting environment +[INFO] [1736215568.702080]: cleared current plan +[INFO] [1736215568.703105]: moving to home +[INFO] [1736215569.705302]: reached home +[INFO] [1736215569.708023]: reset action history +[INFO] [1736215572.415623]: environment reset +Starting episode 0 +[INFO] [1736215572.418081]: resetting environment +[INFO] [1736215572.420788]: cleared current plan +[INFO] [1736215572.421615]: moving to home +[INFO] [1736215573.424059]: reached home +[INFO] [1736215573.425520]: reset action history +[INFO] [1736215576.133769]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736215583.370021]: resetting environment +[INFO] [1736215583.372816]: cleared current plan +[INFO] [1736215583.373781]: moving to home +[INFO] [1736215584.376412]: reached home +[INFO] [1736215584.377627]: reset action history +[INFO] [1736215587.085607]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/episode_rosbags/episode_0_2025-01-06-21-06-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2118796.log +started roslaunch server http://158.130.50.37:35929/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2119062] +process[print_text-2]: started with pid [2119063] +process[rosbag_record-3]: started with pid [2119087] +started bagging! +[INFO] [1736215614.123756]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736215614.844782]: Stored episode 1. +[INFO] [1736215614.845209]: Saving video... +[INFO] [1736215615.043408]: Video saved! +Starting episode 1 +[INFO] [1736215615.054758]: resetting environment +[INFO] [1736215615.062360]: cleared current plan +[INFO] [1736215615.063802]: moving to home +[INFO] [1736215618.468155]: reached home +[INFO] [1736215618.468614]: reset action history +[INFO] [1736215621.176816]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215624.989492]: resetting environment +[INFO] [1736215624.996578]: cleared current plan +[INFO] [1736215624.997200]: moving to home +[INFO] [1736215625.998676]: reached home +[INFO] [1736215625.999088]: reset action history +[INFO] [1736215628.706180]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/episode_rosbags/episode_1_2025-01-06-21-07-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2118796.log +started roslaunch server http://158.130.50.37:43459/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2119197] +process[print_text-5]: started with pid [2119198] +process[rosbag_record-6]: started with pid [2119222] +started bagging! +For topic gripper_width: timestamp difference is 110809593 for nearest: 1736215648299834041 - target: 1736215648189024448 at idx 249 +For topic gripper_width: timestamp difference is 113200226 for nearest: 1736215651099880734 - target: 1736215650986680508 at idx 249 +[INFO] [1736215655.682157]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736215656.422359]: Stored episode 2. +[INFO] [1736215656.423021]: Saving video... +[INFO] [1736215656.924754]: Video saved! +Starting episode 2 +[INFO] [1736215656.966284]: resetting environment +[INFO] [1736215656.978805]: cleared current plan +[INFO] [1736215656.979892]: moving to home +[INFO] [1736215660.589929]: reached home +[INFO] [1736215660.592937]: reset action history +[INFO] [1736215663.365017]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215675.269813]: resetting environment +[INFO] [1736215675.285444]: cleared current plan +[INFO] [1736215675.286260]: moving to home +[INFO] [1736215676.289393]: reached home +[INFO] [1736215676.290437]: reset action history +[INFO] [1736215678.998997]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/episode_rosbags/episode_2_2025-01-06-21-07-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2118796.log +started roslaunch server http://158.130.50.37:35493/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2119331] +process[print_text-8]: started with pid [2119332] +process[rosbag_record-9]: started with pid [2119356] +started bagging! +For topic gripper_width: timestamp difference is 103583401 for nearest: 1736215680399880713 - target: 1736215680296297312 at idx 19 +For topic gripper_width: timestamp difference is 104241353 for nearest: 1736215681199881059 - target: 1736215681095639706 at idx 28 +For topic gripper_width: timestamp difference is 104876334 for nearest: 1736215681999881560 - target: 1736215681895005226 at idx 41 +For topic gripper_width: timestamp difference is 107318247 for nearest: 1736215684999882305 - target: 1736215684892564058 at idx 78 +For topic gripper_width: timestamp difference is 108083167 for nearest: 1736215685999841371 - target: 1736215685891758204 at idx 90 +For topic gripper_width: timestamp difference is 108895578 for nearest: 1736215686800005235 - target: 1736215686691109657 at idx 99 +For topic gripper_width: timestamp difference is 109426136 for nearest: 1736215687599878664 - target: 1736215687490452528 at idx 112 +For topic gripper_width: timestamp difference is 112714904 for nearest: 1736215691599910396 - target: 1736215691487195492 at idx 172 +For topic gripper_width: timestamp difference is 113837380 for nearest: 1736215692999905247 - target: 1736215692886067867 at idx 189 +For topic gripper_width: timestamp difference is 116389229 for nearest: 1736215696199837401 - target: 1736215696083448172 at idx 228 +For topic gripper_width: timestamp difference is 117068450 for nearest: 1736215696999869506 - target: 1736215696882801056 at idx 235 +For topic gripper_width: timestamp difference is 121074507 for nearest: 1736215701999863932 - target: 1736215701878789425 at idx 249 +For topic gripper_width: timestamp difference is 121199686 for nearest: 1736215702199830372 - target: 1736215702078630686 at idx 249 +For topic gripper_width: timestamp difference is 122771671 for nearest: 1736215704199879577 - target: 1736215704077107906 at idx 249 +[INFO] [1736215705.968458]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit[pub_text-7] killing on exit +[INFO] [1736215706.767987]: Stored episode 3. +[INFO] [1736215706.769307]: Saving video... +[INFO] [1736215707.125355]: Video saved! +Starting episode 3 +[INFO] [1736215707.176838]: resetting environment +[INFO] [1736215707.190852]: cleared current plan +[INFO] [1736215707.191403]: moving to home +[WARN] [1736215707.138180]: Plan exhausted +[WARN] [1736215707.176264]: Plan exhausted +[INFO] [1736215710.596486]: reached home +[INFO] [1736215710.614499]: reset action history +[INFO] [1736215713.446244]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215717.611104]: resetting environment +[INFO] [1736215717.624928]: cleared current plan +[INFO] [1736215717.626240]: moving to home +[INFO] [1736215718.630883]: reached home +[INFO] [1736215718.633217]: reset action history +[INFO] [1736215721.546755]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/episode_rosbags/episode_3_2025-01-06-21-08-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2118796.log +started roslaunch server http://158.130.50.37:35481/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2119471] +process[print_text-11]: started with pid [2119472] +process[rosbag_record-12]: started with pid [2119496] +started bagging! +[INFO] [1736215748.528315]: Storing episode... +[pub_text-10] killing on exit +[print_text-11] killing on exit[rosbag_record-12] killing on exit +[INFO] [1736215749.314934]: Stored episode 4. +[INFO] [1736215749.320418]: Saving video... +[INFO] [1736215749.761075]: Video saved! +Starting episode 4 +[INFO] [1736215749.807367]: resetting environment +[INFO] [1736215749.837096]: cleared current plan +[INFO] [1736215749.842191]: moving to home +[INFO] [1736215753.259079]: reached home +[INFO] [1736215753.264412]: reset action history +[INFO] [1736215756.044795]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736215762.608289]: resetting environment +[INFO] [1736215762.630900]: cleared current plan +[INFO] [1736215762.633179]: moving to home +[INFO] [1736215763.643953]: reached home +[INFO] [1736215763.646310]: reset action history +[INFO] [1736215766.429259]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/episode_rosbags/episode_4_2025-01-06-21-09-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2118796.log +started roslaunch server http://158.130.50.37:35483/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2119627] +process[print_text-14]: started with pid [2119628] +process[rosbag_record-15]: started with pid [2119629] +started bagging! +For topic gripper_width: timestamp difference is 113792843 for nearest: 1736215775699881500 - target: 1736215775586088657 at idx 129 +For topic gripper_width: timestamp difference is -112955322 for nearest: 1736215783533213148 - target: 1736215783646168470 at idx 239 +[INFO] [1736215793.674571]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736215794.442174]: Stored episode 5. +[INFO] [1736215794.449585]: Saving video... +[INFO] [1736215794.827660]: Video saved! +[INFO] [1736215794.877121]: resetting environment +[INFO] [1736215794.900642]: cleared current plan +[INFO] [1736215794.903190]: moving to home +[INFO] [1736215798.318270]: reached home +[INFO] [1736215798.322077]: reset action history +[INFO] [1736215801.177523]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..19c434874e4403ce05d885e23b0dac030af0b1f1 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T02:06:00.423214", + "startedAt": "2025-01-07T02:05:59.919242", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.779375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.025, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 545.061149597168 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e09ebc5f258b802b65e7a1fd115cee422bcbd6 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6960c47621db1ed25f24b27e4c1ac777c82b6889a68ffd14ab674a6733b385e1", "size": 1134395, "path": "media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4"}, "global_step": 664, "_timestamp": 1736215804.1029203, "_runtime": 244.17179226875305, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "b2e2b173de6e4d2512f5dd083c82685dd82fbb0ca632f6345727468c5f48ccac", "size": 1192512, "path": "media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4"}, "eval/num_success": 1.0, "episode": 35.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "daf7d1cec72ac69f865682a674836bcec8d12414f0914f8c0a5d7a662b9f06f7", "size": 1130234, "path": "media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "2898cfd9388e3dddd2d6aae0bcde25d9b76ff152c17d68c983c8f9d0ffb2c87a", "size": 1133700, "path": "media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "635826221e5f85ee234a0ccd9dbcd1e1bf368c7e54ca2949f80ca6b5f9a8b145", "size": 1213859, "path": "media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://dn2z90kyxg6hhk74zactgkhfuzo944ztxp53ve9s83h9ib3ns9cq2jn86j1b7rd0dy7i6k0xano1vr1q3in5z578ws3x5gfy3njfhhilcauzvq64alj8ub71iiqzw2i4/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://h8kude4pgovsooojzkkuuqn97togesqn4rah4r896fwvmu9jqp5746q7jxt72fuqf7yk4v1hg33d1hp20bsbfilkv2f67vhsqwr8uch77bcna6dwiaqocb562s62ghez:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 35, "total_success_rate": 0.5714285714285714, "total_success_rate_lb_0.01": 0.3716529608029522, "total_success_rate_ub_0.01": 0.7538510262458453, "total_success_rate_lb_0.025": 0.410685271042567, "total_success_rate_ub_0.025": 0.7356967329507549, "total_success_rate_lb_0.05": 0.4360231012233896, "total_success_rate_ub_0.05": 0.6986087858279691, "total_success_rate_lb_0.1": 0.4541952274831351, "total_success_rate_ub_0.1": 0.6842993311215397, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "51a31336f9897b8b206f5f40b68b24414baba81b9933e86c5999ebc818195da9", "size": 519, "artifact_path": "wandb-client-artifact://1yzslf5g120sjxzc7ztrahosatxek320mb9qd2y2zokdp5vll7hj97p46oz90wckvnxacdosmtk84ptvvt74oyjeeqbejoa4gub3p10bzefveavpynserdilp3f6yxu9/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cblijuf8s10mg60sifl7bntesedu9lg1tkakaxw10x7fsl57nwg11i1qgfgl7gs59smjvnmd98bbed2kiw358jkvdokvsm07o3dbms0zcurhialrzmmqzyelhghrfojw:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 248}, "eval/total_success_rate": 0.5714285969734192, "eval/total_success_rate_lb_0.01": 0.3716529607772827, "eval/total_success_rate_ub_0.01": 0.7538509964942932, "eval/total_success_rate_lb_0.025": 0.410685271024704, "eval/total_success_rate_ub_0.025": 0.7356967329978943, "eval/total_success_rate_lb_0.05": 0.43602311611175537, "eval/total_success_rate_ub_0.05": 0.6986087560653687, "eval/total_success_rate_lb_0.1": 0.4541952311992645, "eval/total_success_rate_ub_0.1": 0.6842993497848511} \ No newline at end of file diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug-internal.log b/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..657276e33e3895c81132f93eb1b0622aea0d3c33 --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug-internal.log @@ -0,0 +1,750 @@ +2025-01-06 21:05:59,930 INFO StreamThr :2118961 [internal.py:wandb_internal():85] W&B internal server running at pid: 2118961, started at: 2025-01-06 21:05:59.930454 +2025-01-06 21:05:59,931 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status +2025-01-06 21:05:59,944 INFO WriterThread:2118961 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb +2025-01-06 21:05:59,944 DEBUG SenderThread:2118961 [sender.py:send():379] send: header +2025-01-06 21:05:59,952 DEBUG SenderThread:2118961 [sender.py:send():379] send: run +2025-01-06 21:06:00,233 INFO SenderThread:2118961 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files +2025-01-06 21:06:00,233 INFO SenderThread:2118961 [sender.py:_start_run_threads():1188] run started: c9fefav0 with start time 1736215559.931128 +2025-01-06 21:06:00,254 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 21:06:00,254 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: check_version +2025-01-06 21:06:00,405 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 21:06:00,410 DEBUG HandlerThread:2118961 [system_info.py:__init__():26] System info init +2025-01-06 21:06:00,410 DEBUG HandlerThread:2118961 [system_info.py:__init__():41] System info init done +2025-01-06 21:06:00,410 INFO HandlerThread:2118961 [system_monitor.py:start():194] Starting system monitor +2025-01-06 21:06:00,410 INFO SystemMonitor:2118961 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 21:06:00,410 INFO HandlerThread:2118961 [system_monitor.py:probe():214] Collecting system info +2025-01-06 21:06:00,411 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 21:06:00,411 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started disk monitoring +2025-01-06 21:06:00,412 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 21:06:00,412 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started memory monitoring +2025-01-06 21:06:00,413 INFO SystemMonitor:2118961 [interfaces.py:start():188] Started network monitoring +2025-01-06 21:06:00,423 DEBUG HandlerThread:2118961 [system_info.py:probe():152] Probing system +2025-01-06 21:06:00,424 DEBUG HandlerThread:2118961 [system_info.py:_probe_git():137] Probing git +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:_probe_git():145] Probing git done +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:probe():200] Probing system done +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T02:06:00.423214', 'startedAt': '2025-01-07T02:05:59.919242', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.779375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.025, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 545.061149597168}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 21:06:00,429 INFO HandlerThread:2118961 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 21:06:00,429 INFO HandlerThread:2118961 [system_monitor.py:probe():227] Publishing system info +2025-01-06 21:06:00,429 DEBUG HandlerThread:2118961 [system_info.py:_save_code():44] Saving code +2025-01-06 21:06:00,431 DEBUG HandlerThread:2118961 [system_info.py:_save_code():67] Saving code done +2025-01-06 21:06:00,431 DEBUG HandlerThread:2118961 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 21:06:00,493 DEBUG HandlerThread:2118961 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 21:06:00,494 INFO HandlerThread:2118961 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 21:06:00,497 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:00,497 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 21:06:00,497 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 21:06:00,498 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 21:06:00,722 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/d02znp1c-wandb-metadata.json +2025-01-06 21:06:00,726 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 21:06:00,726 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: python_packages +2025-01-06 21:06:00,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:00,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:00,729 INFO HandlerThread:2118961 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/tb" + save: true +} +control { + relay_id: "b39b9ea63fe7412f80c74c56999bbc9f" +} +_info { + stream_id: "c9fefav0" +} + +2025-01-06 21:06:00,730 DEBUG HandlerThread:2118961 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 21:06:00,759 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/0kw4w1kr-code/FISH/eval_robot.py +2025-01-06 21:06:00,775 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/5zpoosz0-diff.patch +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: telemetry +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: tbrecord +2025-01-06 21:06:00,832 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:00,832 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 with policy live +2025-01-06 21:06:01,011 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/1tzoylhq-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch +2025-01-06 21:06:01,235 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt +2025-01-06 21:06:01,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH +2025-01-06 21:06:01,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code +2025-01-06 21:06:01,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:02,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:03,234 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:03,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:04,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:05,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:05,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:06,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:07,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:08,698 DEBUG SenderThread:2118961 [sender.py:send():379] send: config +2025-01-06 21:06:08,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:09,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:10,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:11,236 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:11,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:11,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:12,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:13,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:14,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:15,237 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:15,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:15,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:16,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:17,177 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:17,238 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:17,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:18,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:19,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:20,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:21,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:22,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:22,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:23,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:24,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:25,240 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:25,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:26,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:27,241 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:27,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:28,393 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:28,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:29,241 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:29,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:30,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:30,727 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:30,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:31,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:32,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:33,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:33,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:34,243 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:06:34,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:35,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:36,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:37,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:38,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:39,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:39,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:40,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:41,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:42,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:43,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:44,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:45,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:06:45,727 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:06:45,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:45,779 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:46,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:47,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:48,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:49,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:50,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:51,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:51,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:52,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:53,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:54,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:55,053 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:06:55,053 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 with policy now +2025-01-06 21:06:55,054 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:06:55,055 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:06:55,055 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:06:55,055 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:06:55,248 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:06:55,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media +2025-01-06 21:06:55,420 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/r23wablw-media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:55,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:55,759 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/0u9q9fb7-media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:06:56,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:06:56,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:57,249 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:06:57,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:58,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:06:59,734 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:00,413 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 21:07:00,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:07:00,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:00,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:01,250 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:01,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:02,221 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:02,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:03,251 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:03,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:04,735 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:05,252 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:05,484 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mg199rqz-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:05,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:06,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:07,253 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:07,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:08,041 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:08,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:09,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:10,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:11,254 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:11,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:12,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:13,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:13,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:14,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:15,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:15,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:16,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:17,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:18,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:18,838 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:19,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:20,737 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:21,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:22,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:23,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:24,730 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:24,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:25,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:26,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:27,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:28,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:29,260 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:29,739 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:30,345 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:30,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:07:30,727 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:30,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:30,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:31,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:32,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:33,261 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:33,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:34,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:35,682 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:35,741 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:36,747 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:36,969 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:07:36,969 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:07:36,969 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 with policy now +2025-01-06 21:07:36,970 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:07:36,970 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:07:36,971 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:07:37,262 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:07:37,263 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/3l9ox851-media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,558 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/8371k7ey-media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:07:37,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:38,750 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:39,263 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:39,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:40,738 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:40,758 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:41,755 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:42,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:43,264 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:43,748 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:44,748 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:45,265 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:45,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:07:45,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:07:45,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:45,876 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:46,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:47,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:48,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:49,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:50,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:51,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:51,749 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:52,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:53,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:54,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:55,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:55,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:56,140 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:07:56,141 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:07:56,141 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:07:56,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:07:56,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:07:56,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:57,269 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:07:57,333 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:07:57,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:58,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:07:59,752 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:00,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:08:00,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:00,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:00,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:01,271 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:01,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:02,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:03,061 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:03,271 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:03,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:04,754 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:05,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:06,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:07,273 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:07,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:08,601 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:08,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:09,274 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:09,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:10,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:11,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:12,765 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:13,275 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:13,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:14,057 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:14,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:15,276 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:15,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:15,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:15,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:16,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:17,276 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:17,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:18,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:19,277 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:19,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:20,042 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:20,766 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:21,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:22,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:23,278 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:23,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:24,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:25,241 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:25,279 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:25,767 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:26,774 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:27,169 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:08:27,170 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 with policy now +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:08:27,170 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:08:27,170 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,280 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:08:27,472 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/71vu78cf-media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,769 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/vebbpivl-media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:08:27,769 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:28,773 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:29,281 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:29,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:30,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:08:30,415 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:30,733 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:30,733 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:30,772 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:31,780 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:32,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:33,282 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:33,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:34,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:35,283 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:35,493 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:35,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:36,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:37,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:37,993 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:08:37,993 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:08:37,994 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:08:38,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:08:38,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:08:38,528 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/i4f3rr73-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:08:38,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:39,284 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:39,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:40,673 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:40,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:41,285 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:41,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:42,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:43,286 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:08:43,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:44,781 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:45,728 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:08:45,728 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:08:45,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:45,792 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:46,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:47,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:48,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:49,782 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:50,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:51,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:51,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:52,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:53,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:54,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:55,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:56,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:08:56,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:57,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:58,783 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:08:59,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:00,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:09:00,729 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:00,729 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:00,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:01,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:01,844 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:02,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:03,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:04,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:05,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:06,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:07,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:07,784 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:09,294 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:09,798 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:09,803 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:09:09,803 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 with policy now +2025-01-06 21:09:09,845 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:09:09,846 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:09,846 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:09,846 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:10,068 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/2k9al4yi-media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:09:10,591 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/m6cjje27-media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:09:10,792 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:11,295 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:11,791 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:12,743 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:12,794 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:13,793 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:14,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:15,296 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:15,740 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:15,741 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:15,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:16,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:17,297 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:17,800 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:18,093 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:18,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:19,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:20,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:21,802 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:22,810 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:23,047 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:23,047 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:23,047 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:23,299 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:09:23,645 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:23,818 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:24,816 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:25,300 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:25,811 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:26,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:27,301 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:27,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:28,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:28,963 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:29,301 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:29,804 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:30,416 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:09:30,731 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:30,732 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:30,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:31,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:32,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:33,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:34,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:34,825 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:35,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:36,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:37,305 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:37,805 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:38,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:39,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:40,744 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:40,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:41,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:42,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:43,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:44,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:45,308 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:45,732 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:09:45,732 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:09:45,806 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:45,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:46,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:47,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:48,807 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:49,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:50,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:51,744 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:51,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:52,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:53,808 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:54,823 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:54,875 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:09:54,875 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 with policy now +2025-01-06 21:09:54,876 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:09:54,876 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:09:54,876 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:09:54,877 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:09:55,136 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/s6cpd9mv-media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval +2025-01-06 21:09:55,620 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/7somprq8-media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:09:55,823 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:56,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:09:56,825 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:57,312 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:57,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:58,827 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:09:59,313 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:09:59,846 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:00,416 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:10:00,736 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 21:10:00,736 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: stop_status +2025-01-06 21:10:00,832 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:01,848 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:02,221 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:02,839 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:03,315 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:03,315 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:03,557 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mzr8ew5q-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:03,568 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:10:03,569 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:10:03,753 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:10:03,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:04,026 INFO wandb-upload_0:2118961 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpil_x2tbd +2025-01-06 21:10:04,199 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 21:10:04,206 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:04,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media +2025-01-06 21:10:04,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval +2025-01-06 21:10:04,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table +2025-01-06 21:10:04,413 INFO SenderThread:2118961 [sender.py:send_request_log_artifact():1525] logged artifact run-c9fefav0-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM4MjE4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2OTMzNw==', 'latestArtifact': None}} +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:10:04,414 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:04,414 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,415 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: telemetry +2025-01-06 21:10:04,415 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:04,416 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,419 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,419 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,420 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,420 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,421 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,421 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,422 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,422 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,423 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,423 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,423 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,424 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,424 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,424 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,424 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,425 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,425 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,425 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,425 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,426 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,426 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,426 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:04,426 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:04,427 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 21:10:04,618 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/mxdnn4xs-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:04,619 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/36bzv95d-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:04,804 INFO wandb-upload_0:2118961 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6ue9dgnr +2025-01-06 21:10:04,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:04,861 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/kf1v0fpa-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:05,236 INFO SenderThread:2118961 [sender.py:send_request_log_artifact():1525] logged artifact run-c9fefav0-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjM4MjIxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDY2OTM0NA==', 'latestArtifact': None}} +2025-01-06 21:10:05,236 DEBUG SenderThread:2118961 [sender.py:send():379] send: files +2025-01-06 21:10:05,236 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json with policy now +2025-01-06 21:10:05,237 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:05,237 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:05,237 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:05,316 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:05,317 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:05,476 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/vzfhfdm4-media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:05,830 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:06,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:07,751 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:07,831 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:08,786 DEBUG SenderThread:2118961 [sender.py:send():379] send: exit +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():588] handling runtime: 248 +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:08,786 INFO SenderThread:2118961 [sender.py:send_exit():594] send defer +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 21:10:08,787 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 1 +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 21:10:08,787 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 21:10:08,787 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 2 +2025-01-06 21:10:08,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,787 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 21:10:08,788 INFO HandlerThread:2118961 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 21:10:08,788 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 21:10:08,788 DEBUG SystemMonitor:2118961 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 21:10:08,789 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 21:10:08,789 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 21:10:08,838 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 21:10:08,839 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 21:10:08,839 INFO HandlerThread:2118961 [interfaces.py:finish():200] Joined network monitor +2025-01-06 21:10:08,839 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,839 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 21:10:08,839 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 3 +2025-01-06 21:10:08,839 DEBUG SenderThread:2118961 [sender.py:send():379] send: stats +2025-01-06 21:10:08,840 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,840 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 21:10:08,840 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:08,840 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 21:10:08,840 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 4 +2025-01-06 21:10:08,840 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:08,840 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 21:10:09,318 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:11,319 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:15,107 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,108 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 21:10:15,108 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 5 +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send():379] send: history +2025-01-06 21:10:15,108 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: summary_record +2025-01-06 21:10:15,109 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,110 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:15,110 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 21:10:15,110 DEBUG SenderThread:2118961 [sender.py:send():379] send: summary +2025-01-06 21:10:15,113 INFO SenderThread:2118961 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 21:10:15,114 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 6 +2025-01-06 21:10:15,114 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,114 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 21:10:15,114 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,114 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 21:10:15,117 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 21:10:15,250 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 7 +2025-01-06 21:10:15,250 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:15,250 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 21:10:15,251 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:15,251 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 21:10:15,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:15,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:10:15,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:17,280 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 8 +2025-01-06 21:10:17,281 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:17,281 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:17,281 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 21:10:17,281 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:17,281 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 21:10:17,281 INFO SenderThread:2118961 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 21:10:17,285 INFO SenderThread:2118961 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 21:10:17,287 INFO SenderThread:2118961 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 21:10:17,299 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 9 +2025-01-06 21:10:17,299 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:17,299 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 21:10:17,300 DEBUG SenderThread:2118961 [sender.py:send():379] send: artifact +2025-01-06 21:10:17,321 INFO Thread-12 :2118961 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkwNjU3MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 21:10:17,506 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 21:10:17,506 INFO SenderThread:2118961 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 21:10:17,787 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,322 INFO SenderThread:2118961 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml config.yaml +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json wandb-summary.json +2025-01-06 21:10:18,323 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:18,325 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt requirements.txt +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-metadata.json wandb-metadata.json +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log output.log +2025-01-06 21:10:18,326 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/diff.patch diff.patch +2025-01-06 21:10:18,327 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 media/videos/eval/2_eval_2_daf7d1cec72ac69f8656.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 media/videos/eval/0_eval_0_6960c47621db1ed25f24.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 media/videos/eval/3_eval_3_2898cfd9388e3dddd2d6.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 media/videos/eval/1_eval_1_b2e2b173de6e4d2512f5.mp4 +2025-01-06 21:10:18,331 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 media/videos/eval/4_eval_4_635826221e5f85ee234a.mp4 +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json media/table/eval/total_success_rate_ci_6_51a31336f9897b8b206f.table.json +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 10 +2025-01-06 21:10:18,332 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,332 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,332 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 21:10:18,332 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 21:10:18,332 INFO SenderThread:2118961 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:10:18,507 INFO wandb-upload_1:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/config.yaml +2025-01-06 21:10:18,526 INFO wandb-upload_2:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/wandb-summary.json +2025-01-06 21:10:18,532 INFO wandb-upload_0:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/output.log +2025-01-06 21:10:18,549 INFO wandb-upload_3:2118961 [upload_job.py:push():130] Uploaded file /tmp/tmpxx12k9b8wandb/tyb1g2h5-events.out.tfevents.1736215560.leonmkim-ROG-Strix-G15CS-G15CS.2118796.0 +2025-01-06 21:10:18,561 INFO wandb-upload_4:2118961 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/files/requirements.txt +2025-01-06 21:10:18,762 INFO Thread-11 (_thread_body):2118961 [sender.py:transition_state():614] send defer: 11 +2025-01-06 21:10:18,762 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,762 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 21:10:18,762 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,762 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 21:10:18,762 INFO SenderThread:2118961 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 21:10:18,763 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 12 +2025-01-06 21:10:18,764 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,764 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 21:10:18,764 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,764 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 21:10:18,764 INFO SenderThread:2118961 [file_stream.py:finish():601] file stream finish called +2025-01-06 21:10:18,788 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,934 INFO SenderThread:2118961 [file_stream.py:finish():605] file stream finish is done +2025-01-06 21:10:18,934 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 13 +2025-01-06 21:10:18,934 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,934 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,935 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:transition_state():614] send defer: 14 +2025-01-06 21:10:18,935 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: defer +2025-01-06 21:10:18,935 INFO HandlerThread:2118961 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send():379] send: final +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send():379] send: footer +2025-01-06 21:10:18,935 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: defer +2025-01-06 21:10:18,935 INFO SenderThread:2118961 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 21:10:18,936 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 21:10:18,937 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 21:10:18,937 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,937 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 21:10:18,937 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: server_info +2025-01-06 21:10:18,938 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 21:10:18,992 DEBUG SenderThread:2118961 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 21:10:18,993 INFO MainThread:2118961 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 21:10:18,994 DEBUG HandlerThread:2118961 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 21:10:18,994 INFO HandlerThread:2118961 [handler.py:finish():882] shutting down handler +2025-01-06 21:10:19,937 INFO WriterThread:2118961 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [sender.py:finish():1615] shutting down sender +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 21:10:19,993 INFO SenderThread:2118961 [file_pusher.py:join():175] waiting for file pusher diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug.log b/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a2c47ccff05cdc8e6a86b287a2a47909864710ea --- /dev/null +++ b/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Configure stats pid to 2118796 +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/settings +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug.log +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/wandb/run-20250106_210559-c9fefav0/logs/debug-internal.log +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():569] calling init triggers +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0'} +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():619] starting backend +2025-01-06 21:05:59,925 INFO MainThread:2118796 [wandb_init.py:init():623] setting up manager +2025-01-06 21:05:59,929 INFO MainThread:2118796 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 21:05:59,930 INFO MainThread:2118796 [wandb_init.py:init():631] backend started and connected +2025-01-06 21:05:59,943 INFO MainThread:2118796 [wandb_init.py:init():720] updated telemetry +2025-01-06 21:05:59,950 INFO MainThread:2118796 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 21:06:00,254 INFO MainThread:2118796 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 21:06:00,400 INFO MainThread:2118796 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 21:06:00,400 INFO MainThread:2118796 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 21:06:00,727 INFO MainThread:2118796 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 21:06:00,728 INFO MainThread:2118796 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 21:06:00,729 INFO MainThread:2118796 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 21:06:00,729 INFO MainThread:2118796 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/c9fefav0/tb, True +2025-01-06 21:06:08,698 INFO MainThread:2118796 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 21:10:19,994 WARNING MsgRouterThr:2118796 [router.py:message_loop():77] message_loop has been closed diff --git a/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb b/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5c8c668ce00c4fbc27357d08f7cbced395773858 Binary files /dev/null and b/c9fefav0/wandb/run-20250106_210559-c9fefav0/run-c9fefav0.wandb differ diff --git a/fnq3cpdm/episode_rosbags/episode_0_2025-01-06-17-13-55.bag b/fnq3cpdm/episode_rosbags/episode_0_2025-01-06-17-13-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..ca8b8900bf847cd5296ee73fe587bfb780a545b8 --- /dev/null +++ b/fnq3cpdm/episode_rosbags/episode_0_2025-01-06-17-13-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:586f49170a8a5a83e6da1ce63c02853b7761a4dc8192b3c8075eea9811ef756b +size 1402158885 diff --git a/fnq3cpdm/episode_rosbags/episode_1_2025-01-06-17-14-46.bag b/fnq3cpdm/episode_rosbags/episode_1_2025-01-06-17-14-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..aacc9ffdc2d7c5b758e50dcde0e219e844694e65 --- /dev/null +++ b/fnq3cpdm/episode_rosbags/episode_1_2025-01-06-17-14-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96d8d14e08fb2b773a5ab954663d0a0f780ea9bdc017a9c1b9e65b76a703034f +size 1410515109 diff --git a/fnq3cpdm/episode_rosbags/episode_4_2025-01-06-17-17-26.bag b/fnq3cpdm/episode_rosbags/episode_4_2025-01-06-17-17-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..466f583f1d30edd4e2af95f9a377e85474b7e6aa --- /dev/null +++ b/fnq3cpdm/episode_rosbags/episode_4_2025-01-06-17-17-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d35fdbb303afa0c5445569de3528009b4a45006499f923cc6b4110222404d41 +size 1409422739 diff --git a/hle5fmwc/episode_rosbags/episode_0_2025-01-06-20-55-22.bag b/hle5fmwc/episode_rosbags/episode_0_2025-01-06-20-55-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..ff42e1adfcce5f5cdf91fa702de5bdece329de64 --- /dev/null +++ b/hle5fmwc/episode_rosbags/episode_0_2025-01-06-20-55-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0e44adb0770de1e040819fa670ce271a56eeb272454f789a6e2dc9b040586eb +size 1412914531 diff --git a/hle5fmwc/episode_rosbags/episode_1_2025-01-06-20-56-03.bag b/hle5fmwc/episode_rosbags/episode_1_2025-01-06-20-56-03.bag new file mode 100644 index 0000000000000000000000000000000000000000..62a2d96207b94f55fd50ca623d917ba2c393d846 --- /dev/null +++ b/hle5fmwc/episode_rosbags/episode_1_2025-01-06-20-56-03.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1de12fd299fa4e47e93b130501be206f894b8857142b2aa10341535b44f1a754 +size 1411364004 diff --git a/hle5fmwc/episode_rosbags/episode_2_2025-01-06-20-56-44.bag b/hle5fmwc/episode_rosbags/episode_2_2025-01-06-20-56-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..4fae030bedd8794aea4ce367dc811eda1cd52e2f --- /dev/null +++ b/hle5fmwc/episode_rosbags/episode_2_2025-01-06-20-56-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:986d283db111ba8a37f9b05e7b7d38982502ff0eeea3f173eeb61b76ba3796bc +size 1409947855 diff --git a/hle5fmwc/episode_rosbags/episode_3_2025-01-06-20-57-23.bag b/hle5fmwc/episode_rosbags/episode_3_2025-01-06-20-57-23.bag new file mode 100644 index 0000000000000000000000000000000000000000..5e3dc3d7e75e54b64709aac323898cbe1fc87aef --- /dev/null +++ b/hle5fmwc/episode_rosbags/episode_3_2025-01-06-20-57-23.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ad6b3b6aa9da6ff8a517247383d575ce71fcdd75aa593af9d48a379b87007f +size 1411895266 diff --git a/hle5fmwc/episode_rosbags/episode_4_2025-01-06-20-58-20.bag b/hle5fmwc/episode_rosbags/episode_4_2025-01-06-20-58-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..6e4f9c02cd6082bf72e1c5b877692cc9a92b4e83 --- /dev/null +++ b/hle5fmwc/episode_rosbags/episode_4_2025-01-06-20-58-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b5d26e2e534d67b51e213eac668825d27c7190daa9635cced05608d6dddc79e +size 1415964315 diff --git a/kccehv5k/episode_rosbags/episode_0_2024-12-31-20-47-35.bag b/kccehv5k/episode_rosbags/episode_0_2024-12-31-20-47-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..fae4f87aaa9153cb1e5115bd892054fcfd1cdb24 --- /dev/null +++ b/kccehv5k/episode_rosbags/episode_0_2024-12-31-20-47-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b16712c41c21118a41a0fc73c2ca6d76895899499bf870b2b1fb5b37fc1e0127 +size 1406897480 diff --git a/opf0nkwg/.hydra/config.yaml b/opf0nkwg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a56df077cebbbb54879d37edcc83ff7d1e7c8eb --- /dev/null +++ b/opf0nkwg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/opf0nkwg/.hydra/hydra.yaml b/opf0nkwg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59aa6ee973bd8d17af735773a7cec0d3956ee656 --- /dev/null +++ b/opf0nkwg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/opf0nkwg/.hydra/overrides.yaml b/opf0nkwg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/opf0nkwg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/opf0nkwg/episode_rosbags/color_K.npy b/opf0nkwg/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/opf0nkwg/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/opf0nkwg/episode_rosbags/episode_0_2025-01-07-22-29-47.bag b/opf0nkwg/episode_rosbags/episode_0_2025-01-07-22-29-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..b9548b128f76d0d2a81cefe6f6cd28670bd248e9 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_0_2025-01-07-22-29-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11aa9ed9d3f67a57b6521b25f3f4138b0444ee1a829441064184c90d28841031 +size 1416706070 diff --git a/opf0nkwg/episode_rosbags/episode_10_2025-01-07-22-37-20.bag b/opf0nkwg/episode_rosbags/episode_10_2025-01-07-22-37-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..141bfc851253cab9a55da9468271b5da98c92848 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_10_2025-01-07-22-37-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80777145dfefda8860943aef4a9c55422708d3ad9cb2abe07b79d55ff853df6f +size 1410216210 diff --git a/opf0nkwg/episode_rosbags/episode_13_2025-01-07-22-39-36.bag b/opf0nkwg/episode_rosbags/episode_13_2025-01-07-22-39-36.bag new file mode 100644 index 0000000000000000000000000000000000000000..49eec7a43e76725f599cba57ec3276b6ddf1a2b1 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_13_2025-01-07-22-39-36.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:022b2703816a3b391b35a1b72d1c3756c0ecaf1d71ecebcaad948f7be9f693db +size 1413090644 diff --git a/opf0nkwg/episode_rosbags/episode_14_2025-01-07-22-40-45.bag b/opf0nkwg/episode_rosbags/episode_14_2025-01-07-22-40-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..4f8210888cb469f1a6647e6501bf3f308ee4d902 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_14_2025-01-07-22-40-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3734b34d9c2ad52d274ff769af10d3477b53f82558091e53f29dcd21af328927 +size 1417693015 diff --git a/opf0nkwg/episode_rosbags/episode_15_2025-01-07-22-41-37.bag b/opf0nkwg/episode_rosbags/episode_15_2025-01-07-22-41-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..6b4ef08771db197f623a23f477d0047225169b13 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_15_2025-01-07-22-41-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:607777e94d62a0df8a32e531267aebba153ac9c006ddf48cc4ab8ef488c3bf3f +size 1412693252 diff --git a/opf0nkwg/episode_rosbags/episode_16_2025-01-07-22-42-29.bag b/opf0nkwg/episode_rosbags/episode_16_2025-01-07-22-42-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..fcc3f4f3bebdde8c8f751bd1f335b18cb6c5673f --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_16_2025-01-07-22-42-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc4277665666ea6632cb447f301c8359f69a4e3e0ebc56517fd174208aeccad +size 1416111542 diff --git a/opf0nkwg/episode_rosbags/episode_17_2025-01-07-22-43-24.bag b/opf0nkwg/episode_rosbags/episode_17_2025-01-07-22-43-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..b39b5b60da783f1b4f8f38727f3f6d62fcc7af1d --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_17_2025-01-07-22-43-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c32487155d08b231b4f76f2e0e4bb3e77558ff83676e211a59badb6a27ec1a +size 1411704721 diff --git a/opf0nkwg/episode_rosbags/episode_18_2025-01-07-22-44-20.bag b/opf0nkwg/episode_rosbags/episode_18_2025-01-07-22-44-20.bag new file mode 100644 index 0000000000000000000000000000000000000000..c6b99fe875d4a50fceaff2231c74a9c28a84630b --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_18_2025-01-07-22-44-20.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:982dca6b0284136d5c0845499791d54a7f9b42d30174f020020301dc6052d213 +size 1412364972 diff --git a/opf0nkwg/episode_rosbags/episode_19_2025-01-07-22-45-14.bag b/opf0nkwg/episode_rosbags/episode_19_2025-01-07-22-45-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..2b289cd880d0ad1f479652df8ef08bee214e0339 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_19_2025-01-07-22-45-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6eb278c19be96ffcbd6080fb905ebd0e67c1ccf74c16731b62b06a6b6defd63 +size 1412095364 diff --git a/opf0nkwg/episode_rosbags/episode_1_2025-01-07-22-30-39.bag b/opf0nkwg/episode_rosbags/episode_1_2025-01-07-22-30-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..809e5f4d644f0d16b6a3c728cc48964b84b8d87e --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_1_2025-01-07-22-30-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90cfe8cb8007fddc7ad14b3773f2913adff50215fff114ce52554c8762a90c79 +size 1411361284 diff --git a/opf0nkwg/episode_rosbags/episode_3_2025-01-07-22-32-04.bag b/opf0nkwg/episode_rosbags/episode_3_2025-01-07-22-32-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..470384fd9aa9de61e1e151bbd65efbe32730ef0d --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_3_2025-01-07-22-32-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381df575c8336bcbc1a40662ff0275cbaed5b56f48d0e3eca6b0c278bbf29bec +size 1409349362 diff --git a/opf0nkwg/episode_rosbags/episode_6_2025-01-07-22-34-18.bag b/opf0nkwg/episode_rosbags/episode_6_2025-01-07-22-34-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..f2ac6d79adcc1e3ee9eb3dfc9e2165b9f71d53d0 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_6_2025-01-07-22-34-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e816236fe5da1ba25c310e9fdb548e0fc73f66ddec2fdb1ab78f13ee5154b9c3 +size 1411290339 diff --git a/opf0nkwg/episode_rosbags/episode_7_2025-01-07-22-35-02.bag b/opf0nkwg/episode_rosbags/episode_7_2025-01-07-22-35-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..628fe3c7c17ea50f55a1cc42131b176f4ba45e4e --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_7_2025-01-07-22-35-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0560919d45a7837b84a96a2e918c2f670bc7957ba707720b95f5346c4716ab6b +size 1414439987 diff --git a/opf0nkwg/episode_rosbags/episode_8_2025-01-07-22-35-50.bag b/opf0nkwg/episode_rosbags/episode_8_2025-01-07-22-35-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..931f752b8079aeb4e1868005b5fe71f7d11c1d2b --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_8_2025-01-07-22-35-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e873cfd4a3bbfac1193759be38827028ec806ac83ad0a3c09b82e3fe7b91ded3 +size 1409001002 diff --git a/opf0nkwg/episode_rosbags/episode_9_2025-01-07-22-36-34.bag b/opf0nkwg/episode_rosbags/episode_9_2025-01-07-22-36-34.bag new file mode 100644 index 0000000000000000000000000000000000000000..cb61e07775a33077197f08b338bd607099196474 --- /dev/null +++ b/opf0nkwg/episode_rosbags/episode_9_2025-01-07-22-36-34.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:829a986f1617322fcf986a404e5d5e74a9407e243b6c5cfe697b94bb1b583202 +size 1411209251 diff --git a/opf0nkwg/eval_video/0_eval.mp4 b/opf0nkwg/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08b03f344809c309de6c4d2eef96bca9ef7bf34b --- /dev/null +++ b/opf0nkwg/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0416889a3e77e5f6820e3f79a7e68b35b8280549e19c319b2f836a6191ef8a3 +size 1212200 diff --git a/opf0nkwg/eval_video/10_eval.mp4 b/opf0nkwg/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39a8e2c6e44cfd615d373f3cbef5ce779a558527 --- /dev/null +++ b/opf0nkwg/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119d64d622b2c3c1d8e6221298783f2fabf5f0bc6232a1dc0dda101bfb9ec6b8 +size 1295177 diff --git a/opf0nkwg/eval_video/11_eval.mp4 b/opf0nkwg/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5eb9f379a0c39892bb6ac256c94286e4b255921d --- /dev/null +++ b/opf0nkwg/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15ed0619cc72d1cf6a653f631fdd6f13518cb33c0f5ea840a376904f2fcff7f +size 1216209 diff --git a/opf0nkwg/eval_video/12_eval.mp4 b/opf0nkwg/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..adb4b744531c57e5b07a15f9b171c3da3b5338a2 --- /dev/null +++ b/opf0nkwg/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d95a49b894bdfcd37b0c1c1669b8c6a141289459d04683071eb8ce38d88d77 +size 1203863 diff --git a/opf0nkwg/eval_video/13_eval.mp4 b/opf0nkwg/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3561c9594dae3dbddaaf254d76a34df8686bb2b3 --- /dev/null +++ b/opf0nkwg/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a7294f80db38ff7cb0178d8dff109458f6d59c9a5dc22995c9c82851c4a719 +size 1433965 diff --git a/opf0nkwg/eval_video/14_eval.mp4 b/opf0nkwg/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..219c0c53d1bd933fb7e768c0d4124ba70314f87e --- /dev/null +++ b/opf0nkwg/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af06c0a0e058587490fe17759e07794abebc17c35366d1a1e6fb428d55f3fe4 +size 1448485 diff --git a/opf0nkwg/eval_video/15_eval.mp4 b/opf0nkwg/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e71cb82fc50db8b8b57f5c1deff94f3d0c4a7761 --- /dev/null +++ b/opf0nkwg/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de33546b235fc7477b2e4fccf32b2a5e8c12b207d50c7c65d2e8298fe04dcff6 +size 1475418 diff --git a/opf0nkwg/eval_video/16_eval.mp4 b/opf0nkwg/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d3ccb87cee91ca799d6db5ab8d01b4ca156161dc --- /dev/null +++ b/opf0nkwg/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:991c282f4f03c1a3f74b839ca93393f883f7c889c8cdf9cb3ad22d513486d439 +size 1560712 diff --git a/opf0nkwg/eval_video/17_eval.mp4 b/opf0nkwg/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37721556c6fd12aabe5487dccaa66911bf019abe --- /dev/null +++ b/opf0nkwg/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf814c17374e1cd6bef0bc2afa3e1ef31988cdfe8e9f9de2860916e6dbb266c +size 1498337 diff --git a/opf0nkwg/eval_video/18_eval.mp4 b/opf0nkwg/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d9450eb119df72ba89b1039cec5992da8d8556e --- /dev/null +++ b/opf0nkwg/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4987dfaab4d6ad84d0ce99e786a1dc0d0b67c7bf407fa710707d10a0c5614139 +size 1510751 diff --git a/opf0nkwg/eval_video/19_eval.mp4 b/opf0nkwg/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e82206d064d00bcf460aef1e4e7466ff0695bef --- /dev/null +++ b/opf0nkwg/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc271ae74232dc330acb4135e48ae2c481ea4f7e8499217eed6c7aabb081e22 +size 1187563 diff --git a/opf0nkwg/eval_video/1_eval.mp4 b/opf0nkwg/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6b5e13255c10d430d260ff91315498995608124 --- /dev/null +++ b/opf0nkwg/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479971f5e97e50638f221f0625e45daf2e59bd333480512dfb081bb5fe7f0095 +size 1198438 diff --git a/opf0nkwg/eval_video/2_eval.mp4 b/opf0nkwg/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd59baf7f1241f39735a654aacf5e572d3ebf4da --- /dev/null +++ b/opf0nkwg/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75d5fcf827136b5c43bf2d3469c641d32b0cfe6bf0d324a0c654a30c5b4fda8 +size 1159893 diff --git a/opf0nkwg/eval_video/3_eval.mp4 b/opf0nkwg/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74ceb5f08afaccb37a70cc6c8c1e3f82ce0df250 --- /dev/null +++ b/opf0nkwg/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4107d85ee8f655917ae7734e9fc72871bde4d98c147f3d5f8ae2145410178402 +size 1168378 diff --git a/opf0nkwg/eval_video/4_eval.mp4 b/opf0nkwg/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a4b7e7a32d351eb84e685923aafcdc6eedc853d --- /dev/null +++ b/opf0nkwg/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4af638a8cd5e4e155cbb4d7b2dcd013be4455d3a2146012c4a159d161cfdb5 +size 1265054 diff --git a/opf0nkwg/eval_video/5_eval.mp4 b/opf0nkwg/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..75ec30c982154e77cf312184ea607de48bc995dc --- /dev/null +++ b/opf0nkwg/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa2adef9cffa563b3117db168e6fa5630369ec1bf420c8c03f2313c78c61347e +size 1213213 diff --git a/opf0nkwg/eval_video/6_eval.mp4 b/opf0nkwg/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87a6c522f8da27ac1a716a41f5161a421fb85bdb --- /dev/null +++ b/opf0nkwg/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bdc6ce07f76b86e3067e1af08273a2e7a4399c69adc8e37d196402db65b2f3f +size 1217184 diff --git a/opf0nkwg/eval_video/7_eval.mp4 b/opf0nkwg/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31707d85ce65f7100c43b13eabefa83e5fbbc71a --- /dev/null +++ b/opf0nkwg/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94820970d3238dd1d30cad3b7fb5cdc5ff5db8a286232696e147349f717d41a +size 1282027 diff --git a/opf0nkwg/eval_video/8_eval.mp4 b/opf0nkwg/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65d96b2feeeed2bc179e736bef5e0017498f786a --- /dev/null +++ b/opf0nkwg/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5641ff60163a911937b6ca4cb77f87c02fc667fe16b75e90ec6e45c5951960 +size 1186965 diff --git a/opf0nkwg/eval_video/9_eval.mp4 b/opf0nkwg/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3152a7eb1216043b09a00230cae300744aba3761 --- /dev/null +++ b/opf0nkwg/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef5c54b11ebe40dec8b27aff851a0802e98ad3a05e55de1df66231e507a869a9 +size 1213538 diff --git a/opf0nkwg/tb/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 b/opf0nkwg/tb/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 new file mode 100644 index 0000000000000000000000000000000000000000..40b08e3de661954b4ca81d21e9993054ab5e6694 --- /dev/null +++ b/opf0nkwg/tb/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31c5a0892793187ba9d2d81b1dd7a4fb82743b8b120f5cbcbd19ae459fe98624 +size 22784 diff --git a/opf0nkwg/wandb/debug-internal.log b/opf0nkwg/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..99d23c3410ef2f63a4276be3b67f696e38c0ed72 --- /dev/null +++ b/opf0nkwg/wandb/debug-internal.log @@ -0,0 +1,2184 @@ +2025-01-07 22:29:27,696 INFO StreamThr :2243413 [internal.py:wandb_internal():85] W&B internal server running at pid: 2243413, started at: 2025-01-07 22:29:27.696166 +2025-01-07 22:29:27,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status +2025-01-07 22:29:27,707 INFO WriterThread:2243413 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb +2025-01-07 22:29:27,707 DEBUG SenderThread:2243413 [sender.py:send():379] send: header +2025-01-07 22:29:27,716 DEBUG SenderThread:2243413 [sender.py:send():379] send: run +2025-01-07 22:29:27,985 INFO SenderThread:2243413 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files +2025-01-07 22:29:27,985 INFO SenderThread:2243413 [sender.py:_start_run_threads():1188] run started: opf0nkwg with start time 1736306967.696375 +2025-01-07 22:29:28,004 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 22:29:28,004 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: check_version +2025-01-07 22:29:28,082 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 22:29:28,087 DEBUG HandlerThread:2243413 [system_info.py:__init__():26] System info init +2025-01-07 22:29:28,087 DEBUG HandlerThread:2243413 [system_info.py:__init__():41] System info init done +2025-01-07 22:29:28,087 INFO HandlerThread:2243413 [system_monitor.py:start():194] Starting system monitor +2025-01-07 22:29:28,087 INFO SystemMonitor:2243413 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 22:29:28,087 INFO HandlerThread:2243413 [system_monitor.py:probe():214] Collecting system info +2025-01-07 22:29:28,088 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 22:29:28,089 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started disk monitoring +2025-01-07 22:29:28,089 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 22:29:28,090 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started memory monitoring +2025-01-07 22:29:28,090 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started network monitoring +2025-01-07 22:29:28,100 DEBUG HandlerThread:2243413 [system_info.py:probe():152] Probing system +2025-01-07 22:29:28,101 DEBUG HandlerThread:2243413 [system_info.py:_probe_git():137] Probing git +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:_probe_git():145] Probing git done +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:probe():200] Probing system done +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T03:29:28.100501', 'startedAt': '2025-01-08T03:29:27.682555', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.0784999999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.977, 'min': 800.0, 'max': 4700.0}, {'current': 4482.979, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 684.1397895812988}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 22:29:28,106 INFO HandlerThread:2243413 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 22:29:28,106 INFO HandlerThread:2243413 [system_monitor.py:probe():227] Publishing system info +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:_save_code():44] Saving code +2025-01-07 22:29:28,108 DEBUG HandlerThread:2243413 [system_info.py:_save_code():67] Saving code done +2025-01-07 22:29:28,108 DEBUG HandlerThread:2243413 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 22:29:28,170 DEBUG HandlerThread:2243413 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 22:29:28,170 INFO HandlerThread:2243413 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 22:29:28,173 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:29:28,173 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 22:29:28,173 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 22:29:28,174 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 22:29:28,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 22:29:28,407 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: python_packages +2025-01-07 22:29:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:28,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:28,411 INFO HandlerThread:2243413 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/tb" + save: true +} +control { + relay_id: "092b5698c5204f738cc04f390ab07652" +} +_info { + stream_id: "opf0nkwg" +} + +2025-01-07 22:29:28,411 DEBUG HandlerThread:2243413 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 22:29:28,414 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/mlfwd2eb-diff.patch +2025-01-07 22:29:28,436 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9wprl8i7-wandb-metadata.json +2025-01-07 22:29:28,442 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zxwa01co-code/FISH/eval_robot.py +2025-01-07 22:29:28,506 DEBUG SenderThread:2243413 [sender.py:send():379] send: telemetry +2025-01-07 22:29:28,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: tbrecord +2025-01-07 22:29:28,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:29:28,507 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 with policy live +2025-01-07 22:29:28,687 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/raltsj4z-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH +2025-01-07 22:29:28,988 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code +2025-01-07 22:29:29,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:30,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:30,986 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:31,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:32,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:32,802 DEBUG SenderThread:2243413 [sender.py:send():379] send: config +2025-01-07 22:29:32,802 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:33,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:34,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:34,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:35,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:36,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:37,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:38,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:38,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:38,988 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:39,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:40,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:41,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:42,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:42,989 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:43,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:43,407 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:43,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:43,491 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:44,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:44,990 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:45,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:46,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:46,991 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:47,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:48,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:48,991 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:49,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:49,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:50,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:51,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:52,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:53,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:54,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:54,621 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:54,993 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:55,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:56,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:57,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:58,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:58,408 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:58,449 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:59,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:00,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:00,571 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:00,994 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:30:01,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:02,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:03,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:04,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:05,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:06,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:06,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:07,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:08,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:09,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:10,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:11,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:12,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:12,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:13,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:13,408 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:13,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:14,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:15,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:15,759 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:30:15,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:30:15,759 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 with policy now +2025-01-07 22:30:15,760 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:30:15,760 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:30:15,761 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:30:15,980 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/b2g0vvz0-media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:15,998 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:30:15,998 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media +2025-01-07 22:30:16,302 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/gsa7cvl6-media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:16,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:16,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:17,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:17,767 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:18,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:19,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:20,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:21,000 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:21,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:22,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:22,925 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:23,001 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:23,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:24,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:25,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:26,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:27,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:28,090 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 22:30:28,091 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:30:28,092 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:28,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:28,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:29,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:30,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:31,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:32,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:33,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:33,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:34,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:35,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:36,005 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:30:36,251 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/20er5t1z-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:30:36,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:37,005 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:37,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:38,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:39,006 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:39,190 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:39,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:40,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:41,006 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:41,429 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:42,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:43,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:43,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:43,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:44,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:44,537 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:45,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:46,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:47,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:48,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:49,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:50,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:50,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:51,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:52,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:53,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:54,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:55,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:55,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:56,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:57,432 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:58,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:30:58,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:58,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:58,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:59,012 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:59,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:00,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:00,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:01,013 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:01,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:02,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:03,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:04,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:05,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:06,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:06,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:07,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:07,197 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:31:07,197 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 with policy now +2025-01-07 22:31:07,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:31:07,200 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:07,200 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:07,200 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:07,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:07,450 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/60aix6kp-media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:31:08,284 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fk371zr3-media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:09,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:09,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:10,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:11,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:11,661 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:12,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:13,017 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:13,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:13,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:13,445 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:14,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:15,017 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:15,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:16,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:17,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:17,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:18,018 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:31:18,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:18,822 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:18,822 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:18,822 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:19,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:19,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:19,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:20,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:21,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:21,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:22,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:23,020 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:23,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:23,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:24,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:25,021 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:25,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:26,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:27,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:28,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:31:28,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:28,410 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:28,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:28,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:29,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:30,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:31,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:32,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:33,023 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:33,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:34,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:34,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:35,024 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:35,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:36,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:37,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:38,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:39,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:40,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:40,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:41,026 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:41,444 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:42,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:43,029 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:43,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:43,410 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:43,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:44,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:45,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:45,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:46,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:47,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:48,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:49,448 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:50,196 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:31:50,196 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 with policy now +2025-01-07 22:31:50,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:31:50,200 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:50,200 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:50,200 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:50,440 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/knmflj8v-media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:50,455 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:31:51,226 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:51,295 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/a38l9q0o-media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:51,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:52,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:53,033 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:53,463 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:54,461 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:55,033 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:55,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:56,472 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:56,512 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:57,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:58,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:31:58,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:58,412 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:58,467 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:59,034 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:59,472 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:00,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:00,656 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:00,656 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:00,656 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:01,035 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:01,035 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:01,286 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/d6rnbtvx-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:01,470 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:01,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:02,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:03,036 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:03,471 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:04,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:05,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:06,468 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:06,750 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:07,038 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:07,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:08,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:09,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:10,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:11,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:12,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:12,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:13,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:13,411 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:13,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:14,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:15,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:16,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:17,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:17,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:18,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:19,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:20,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:21,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:22,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:23,286 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:23,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:24,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:25,052 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:25,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:26,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:27,053 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:27,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:28,093 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:32:28,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:28,412 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:28,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:28,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:29,054 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:29,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:30,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:31,054 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:31,497 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:32,484 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:32,827 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:32:32,827 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 with policy now +2025-01-07 22:32:32,831 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:32:32,831 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:32,831 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:32,831 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:32:33,059 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zknf0zt1-media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,328 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/j2cl0tip-media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,480 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:34,436 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:34,482 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:35,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:35,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:36,493 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:37,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:37,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:38,491 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:39,503 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:40,021 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:40,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:41,058 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:41,496 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:42,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:43,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:43,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:43,503 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:44,541 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:44,710 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:44,710 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:44,710 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:45,059 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:45,059 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:45,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:45,562 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:46,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:47,060 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:47,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:48,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:49,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:50,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:51,061 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:51,205 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:51,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:52,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:53,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:54,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:55,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:56,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:56,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:57,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:58,093 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:32:58,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:58,413 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:58,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:59,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:00,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:01,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:01,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:02,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:03,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:04,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:05,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:06,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:07,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:07,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:08,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:09,067 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:09,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:10,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:11,068 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:11,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:12,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:13,257 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:13,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:13,413 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:13,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:14,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:15,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:16,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:17,514 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:18,274 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:18,335 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:33:18,335 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 with policy now +2025-01-07 22:33:18,343 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:33:18,347 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:33:18,347 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:33:18,347 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:33:18,512 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:18,584 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/raznu5ux-media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,071 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:33:19,071 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:19,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:33:19,342 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/to5pswpt-media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,514 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:20,518 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:21,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:21,531 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:22,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:23,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:23,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:23,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:24,530 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:25,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:26,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:27,074 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:27,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:28,094 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:33:28,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:28,421 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:28,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:28,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:29,539 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:30,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:31,075 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:33:31,324 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/kz9w931i-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:33:31,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:31,766 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:33:31,766 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:33:31,767 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:33:32,076 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:33:32,537 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:33,076 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:33,543 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:34,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:34,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:35,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:36,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:37,077 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:37,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:38,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:39,078 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:39,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:40,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:40,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:41,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:42,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:43,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:43,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:43,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:44,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:45,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:45,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:46,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:47,546 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:48,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:49,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:50,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:51,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:51,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:52,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:53,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:54,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:55,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:56,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:56,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:57,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:58,094 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:33:58,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:58,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:58,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:59,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:00,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:01,086 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:01,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:02,176 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:02,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:03,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:04,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:04,985 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:34:04,985 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:34:04,985 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 with policy now +2025-01-07 22:34:04,986 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:04,986 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:04,986 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:34:05,279 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/ytldlvrg-media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,550 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/86ht2tnr-media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:06,571 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:07,089 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:07,558 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:08,026 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:08,563 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:09,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:10,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:11,090 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:11,556 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:12,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:13,091 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:13,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:13,423 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:13,423 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:13,579 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:14,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:14,820 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:14,820 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:14,821 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:15,092 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:15,092 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:15,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:16,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:17,093 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:17,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:18,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:18,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:19,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:20,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:21,094 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:21,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:22,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:23,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:24,309 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:24,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:25,096 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:25,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:26,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:27,096 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:27,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:28,095 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:34:28,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:28,415 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:28,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:29,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:29,575 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:30,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:31,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:32,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:33,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:34,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:35,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:35,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:36,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:37,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:38,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:39,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:40,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:40,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:41,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:42,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:43,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:43,416 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:43,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:44,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:45,103 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:45,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:45,703 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:46,594 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:47,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:47,271 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:34:47,271 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 with policy now +2025-01-07 22:34:47,280 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:34:47,281 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:47,281 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:47,281 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:47,492 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fu3ls3bi-media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:47,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:34:48,376 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/qbs2cwev-media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:48,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:49,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:49,578 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:50,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:50,743 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:51,583 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:52,599 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:53,106 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:53,595 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:54,595 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:55,107 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:55,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:56,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:56,677 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:57,597 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:58,095 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:34:58,422 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:58,422 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:58,600 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:58,883 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:58,883 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:58,883 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:59,363 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/cz54zzq4-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:59,599 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:00,617 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:01,109 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:01,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:02,362 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:02,596 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:03,110 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:03,617 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:04,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:05,110 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:05,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:06,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:07,111 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:07,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:07,897 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:08,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:09,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:10,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:11,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:12,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:13,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:13,418 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:13,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:13,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:14,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:15,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:16,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:17,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:18,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:19,115 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:19,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:19,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:20,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:21,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:22,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:23,117 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:23,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:24,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:24,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:25,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:26,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:27,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:28,096 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:35:28,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:28,418 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:28,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:29,501 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:29,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:30,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:31,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:32,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:33,633 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:34,031 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:35:34,031 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 with policy now +2025-01-07 22:35:34,046 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:35:34,046 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:35:34,046 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:35:34,047 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:35:34,273 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/bw7cmt2v-media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,529 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4b7pq9rq-media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:35,093 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:35,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:35,625 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:36,641 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:37,122 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:37,644 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:38,636 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:39,123 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:39,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:40,147 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:40,634 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:41,123 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:41,641 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:42,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:43,424 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:43,424 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:43,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:44,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:45,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:45,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:46,648 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:46,942 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:35:46,942 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:35:46,942 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:35:47,125 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:35:47,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:35:47,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:47,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:48,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:49,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:49,663 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:50,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:51,127 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:51,364 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:51,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:52,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:53,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:54,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:55,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:56,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:56,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:57,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:58,096 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:35:58,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:58,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:58,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:59,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:00,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:01,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:01,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:02,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:03,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:04,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:05,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:06,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:07,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:07,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:08,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:09,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:10,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:11,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:12,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:12,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:13,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:13,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:13,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:14,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:15,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:16,659 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:17,660 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:17,877 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:18,577 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:36:18,577 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 with policy now +2025-01-07 22:36:18,580 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:36:18,580 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:36:18,581 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:36:18,581 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:36:18,684 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:18,812 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/04bg8kel-media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,137 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:36:19,137 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:19,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:36:19,411 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/wil3hctr-media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,682 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:20,671 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:21,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:21,676 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:22,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:23,139 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:23,451 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:23,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:24,713 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:25,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:26,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:27,140 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:27,678 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:28,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:36:28,423 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:28,423 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:28,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:28,684 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:29,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:30,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:30,995 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:36:30,995 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:36:30,996 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:36:31,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:36:31,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:36:31,471 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/6hri6jy4-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:36:31,691 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:32,704 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:33,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:33,719 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:34,461 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:34,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:35,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:36,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:37,144 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:37,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:38,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:39,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:40,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:40,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:41,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:42,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:43,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:43,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:43,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:44,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:45,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:45,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:46,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:47,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:48,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:49,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:50,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:51,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:51,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:52,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:53,152 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:53,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:54,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:55,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:56,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:56,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:57,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:58,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:36:58,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:58,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:58,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:59,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:00,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:01,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:01,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:02,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:03,155 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:03,553 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:37:03,554 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 with policy now +2025-01-07 22:37:03,557 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:37:03,558 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:03,558 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:03,558 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:03,722 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:03,776 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/10063bj9-media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:37:04,426 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/xo1c04lk-media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,711 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:05,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:05,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:06,622 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:06,734 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:07,728 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:08,726 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:09,158 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:09,729 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:10,744 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:11,159 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:11,756 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:12,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:12,732 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:13,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:13,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:13,737 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:14,773 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:15,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:16,049 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:16,049 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:16,049 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:16,160 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:37:16,161 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:16,753 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:17,161 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:17,765 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:17,977 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:18,771 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:19,162 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:19,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:20,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:21,163 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:21,766 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:22,774 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:23,038 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:23,778 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:24,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:25,164 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:25,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:26,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:27,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:28,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:37:28,098 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:28,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:28,446 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:28,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:29,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:30,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:31,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:32,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:33,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:33,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:34,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:35,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:36,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:37,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:38,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:38,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:39,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:40,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:41,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:42,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:43,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:43,446 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:43,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:43,762 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:44,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:45,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:46,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:47,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:48,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:49,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:49,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:50,764 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:51,768 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:52,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:37:52,507 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 with policy now +2025-01-07 22:37:52,511 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:37:52,511 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:52,511 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:52,512 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:52,757 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/kazi6hf6-media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:52,774 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:53,177 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:37:53,446 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/oparcluk-media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:53,795 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:54,485 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:54,777 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:55,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:56,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:57,178 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:57,802 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:58,098 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:37:58,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:58,473 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:58,788 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:59,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:00,009 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:00,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:01,179 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:01,776 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:02,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:03,791 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:04,110 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:04,110 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:04,111 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:04,180 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:04,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:04,454 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/jemmtlhf-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:04,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:05,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:05,798 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:05,829 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:06,804 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:07,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:07,800 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:08,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:09,182 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:09,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:10,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:11,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:11,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:12,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:13,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:13,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:13,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:14,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:15,185 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:15,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:16,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:16,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:17,185 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:17,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:18,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:19,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:20,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:21,187 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:21,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:22,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:22,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:23,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:24,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:25,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:26,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:27,189 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:27,282 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:27,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:28,098 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:38:28,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:28,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:28,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:29,190 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:29,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:30,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:31,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:32,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:32,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:33,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:34,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:35,792 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:36,390 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:38:36,391 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 with policy now +2025-01-07 22:38:36,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:38:36,411 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:36,411 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:36,412 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:36,610 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/w1v9s7ig-media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:36,793 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:37,193 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:38:37,464 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9b6tmmo5-media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:37,811 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:38,490 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:38,834 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:39,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:39,812 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:40,815 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:41,195 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:41,801 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:42,804 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:43,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:43,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:43,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:43,811 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:44,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:45,197 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:45,817 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:46,845 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:47,823 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:48,701 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:48,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:49,167 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:49,168 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:49,168 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:49,198 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:49,198 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:49,851 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:50,841 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:51,199 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:51,846 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:52,850 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:53,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:54,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:54,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:55,201 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:55,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:56,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:57,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:58,099 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:38:58,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:58,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:58,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:59,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:59,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:00,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:01,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:02,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:03,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:04,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:05,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:05,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:06,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:07,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:08,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:09,207 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:09,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:10,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:11,208 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:11,257 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:11,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:12,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:13,209 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:13,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:13,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:13,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:14,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:15,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:16,696 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:16,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:17,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:18,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:19,841 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:20,862 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:21,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:21,441 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:39:21,441 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 with policy now +2025-01-07 22:39:21,449 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:39:21,450 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:39:21,450 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:39:21,450 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:39:21,667 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/rcm81113-media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:21,887 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:39:22,496 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fwy80wuk-media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:22,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:22,883 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:23,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:23,862 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:24,879 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:25,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:26,885 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:27,215 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:27,919 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:27,993 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:28,099 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:39:28,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:28,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:28,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:29,215 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:29,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:30,910 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:31,907 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:32,941 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:33,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:39:33,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:33,222 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:39:33,222 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:39:33,222 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:33,222 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:39:33,962 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:34,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:39:34,951 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:35,218 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:35,973 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:36,945 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:37,218 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:37,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:38,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:38,956 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:39,219 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:39,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:40,965 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:41,955 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:42,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:43,221 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:43,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:43,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:43,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:43,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:44,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:45,222 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:45,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:46,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:47,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:48,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:49,224 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:49,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:49,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:50,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:51,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:52,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:53,226 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:53,951 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:54,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:54,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:55,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:56,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:57,228 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:57,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:58,100 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:39:58,450 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:58,450 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:58,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:59,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:59,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:00,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:01,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:02,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:03,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:04,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:05,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:05,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:06,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:07,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:08,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:09,977 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:10,845 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:10,982 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:11,049 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:40:11,049 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:40:11,050 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 with policy now +2025-01-07 22:40:11,050 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:40:11,050 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:40:11,050 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:40:11,233 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:40:11,288 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/yhc8yu3e-media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,562 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/253o2wip-media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,975 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:13,002 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:13,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:13,494 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:13,494 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:14,001 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:15,003 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:15,235 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:15,999 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:16,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:17,003 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:17,983 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:19,011 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:19,237 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:20,005 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:21,002 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:22,034 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:22,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:23,007 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:24,015 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:25,014 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:26,005 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:27,036 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:28,011 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:28,100 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:40:28,101 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:28,498 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:28,498 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:29,043 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:30,012 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:31,016 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:32,016 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:33,032 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:33,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:34,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:35,054 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:36,027 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:37,054 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:38,036 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:39,066 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:39,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:40,060 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:41,063 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:42,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:42,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:40:42,306 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:40:42,306 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:40:42,308 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:40:42,506 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/09wds7yj-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:40:43,058 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:43,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:40:43,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:43,486 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:43,486 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:44,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:44,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:45,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:46,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:47,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:47,248 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:48,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:49,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:50,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:50,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:51,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:52,106 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:53,117 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:54,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:55,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:55,251 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:56,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:56,157 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:57,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:58,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:58,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:40:58,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:58,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:59,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:00,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:01,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:01,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:02,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:03,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:04,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:05,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:06,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:07,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:07,516 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:08,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:09,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:10,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:11,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:12,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:12,516 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:13,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:13,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:13,477 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:14,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:15,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:16,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:17,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:17,575 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:18,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:19,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:20,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:21,096 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:21,265 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:21,817 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:41:21,817 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 with policy now +2025-01-07 22:41:21,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:41:21,822 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:41:21,822 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:41:21,822 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:41:22,037 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/lcdgfjst-media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,088 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:22,265 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:41:22,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:41:22,533 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/vrjpjmsf-media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,915 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:23,117 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:23,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:24,123 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:25,108 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:26,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:27,109 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:27,267 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:28,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:41:28,102 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:28,127 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:28,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:28,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:29,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:29,268 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:30,122 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:31,112 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:32,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:33,137 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:33,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:34,124 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:34,270 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:41:34,370 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:41:34,370 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:41:34,370 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:41:35,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:35,271 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:41:35,271 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:36,170 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:37,145 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:38,132 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:38,529 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:39,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:39,272 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:40,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:41,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:42,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:43,177 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:43,489 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:43,489 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:43,631 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:44,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:45,148 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:46,157 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:47,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:47,275 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:48,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:49,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:49,519 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:50,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:51,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:52,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:53,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:54,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:54,519 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:55,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:56,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:57,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:58,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:41:58,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:58,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:58,477 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:59,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:59,280 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:59,633 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:00,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:01,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:02,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:03,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:04,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:05,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:05,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:06,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:07,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:08,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:09,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:10,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:10,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:11,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:12,154 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:13,156 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:13,286 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:13,485 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:42:13,485 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 with policy now +2025-01-07 22:42:13,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:13,504 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:13,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:42:13,606 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:42:13,606 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:42:13,607 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:42:13,716 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zvk9rtev-media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:14,207 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:42:14,560 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/p1z3uba1-media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:15,183 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:15,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:15,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:16,172 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:17,181 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:18,187 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:19,183 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:19,289 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:20,221 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:21,212 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:21,281 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:21,290 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:22,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:23,229 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:24,234 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:25,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:25,429 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:42:25,429 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:42:25,430 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:42:26,239 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:26,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:42:26,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:42:26,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:27,256 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:27,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:28,102 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:42:28,235 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:28,536 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:28,536 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:29,240 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:30,243 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:31,253 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:31,294 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:31,833 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:32,282 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:33,252 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:34,261 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:35,260 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:36,255 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:37,262 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:37,587 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:38,259 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:39,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:39,297 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:40,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:41,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:41,298 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:42,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:43,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:43,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:43,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:43,505 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:44,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:45,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:46,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:47,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:48,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:48,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:49,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:50,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:51,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:52,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:53,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:54,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:54,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:55,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:56,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:57,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:58,102 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:42:58,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:58,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:58,509 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:59,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:59,619 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:00,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:01,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:02,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:03,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:04,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:05,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:05,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:06,274 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:07,273 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:07,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:07,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:43:07,340 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:43:07,340 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 with policy now +2025-01-07 22:43:07,381 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:43:07,382 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:43:07,382 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:43:07,382 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:43:07,565 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/r6t0t2ft-media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:08,260 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:08,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:43:08,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:08,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:08,584 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/jagjpk1y-media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:09,278 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:09,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:10,305 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:10,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:10,597 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:11,270 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:12,298 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:12,311 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:13,288 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:13,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:13,528 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:14,271 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:15,296 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:16,102 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:16,312 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:16,313 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:17,297 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:18,309 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:19,321 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:20,294 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:20,314 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:43:20,500 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:43:20,500 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:43:20,501 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:43:20,576 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/oabo0or1-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:43:21,185 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:21,297 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:21,315 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:43:22,302 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:22,315 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:23,318 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:24,316 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:24,319 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:25,331 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:26,307 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:26,317 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:26,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:27,319 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:28,110 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:43:28,331 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:28,529 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:28,529 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:29,355 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:30,370 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:31,361 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:31,672 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:32,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:33,352 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:34,360 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:35,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:36,324 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:36,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:37,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:37,622 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:38,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:39,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:40,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:41,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:42,355 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:43,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:43,515 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:43,515 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:43,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:44,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:45,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:46,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:47,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:48,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:49,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:49,582 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:50,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:51,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:52,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:53,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:54,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:55,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:55,582 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:56,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:57,359 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:58,103 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:43:58,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:58,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:58,517 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:59,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:00,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:00,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:01,385 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:02,368 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:02,607 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:44:02,607 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 with policy now +2025-01-07 22:44:02,632 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:44:02,632 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:02,633 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:02,634 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:02,843 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9t2sw8oy-media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:03,335 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:03,335 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:03,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:44:03,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:03,614 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4l9js92x-media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:04,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:04,395 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:05,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:05,406 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:06,182 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:06,386 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:07,401 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:08,337 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:08,394 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:09,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:10,422 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:11,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:11,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:12,339 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:12,429 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:13,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:13,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:13,545 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:14,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:15,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:16,341 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:44:16,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:16,568 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:16,569 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:16,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:16,569 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:17,341 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:17,437 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:18,342 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:18,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:19,451 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:20,343 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:20,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:21,464 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:21,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:22,344 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:22,464 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:23,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:24,467 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:25,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:26,499 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:26,638 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:27,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:28,104 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:44:28,456 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:28,530 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:28,531 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:29,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:30,347 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:30,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:31,466 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:31,787 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:32,466 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:33,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:34,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:35,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:36,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:37,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:37,614 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:38,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:39,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:40,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:41,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:42,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:42,614 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:43,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:43,524 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:43,524 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:44,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:45,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:46,354 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:46,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:47,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:47,765 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:48,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:49,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:50,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:51,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:52,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:53,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:53,615 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:54,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:55,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:56,486 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:57,048 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:44:57,048 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 with policy now +2025-01-07 22:44:57,052 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:44:57,053 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:57,053 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:57,053 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:57,298 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/vvmyr64k-media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:57,358 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:57,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:57,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:44:57,484 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:57,634 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/ubqtz0wg-media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:58,104 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:44:58,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:58,541 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:58,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:58,552 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:58,637 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:59,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:59,542 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:00,360 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:00,573 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:01,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:02,361 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:02,573 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:03,539 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:03,637 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:04,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:05,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:06,362 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:06,528 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:07,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:08,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:08,656 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:09,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:09,632 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:09,632 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:09,633 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:10,364 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:45:10,364 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:10,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:11,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:12,365 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:12,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:13,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:13,558 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:13,558 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:13,932 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:14,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:15,531 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:16,367 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:16,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:17,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:18,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:19,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:19,624 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:20,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:21,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:22,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:23,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:24,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:25,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:25,623 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:26,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:27,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:28,105 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:45:28,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:28,540 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:28,540 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:29,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:30,373 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:30,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:30,632 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:31,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:32,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:33,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:34,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:35,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:36,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:36,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:37,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:38,376 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:38,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:39,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:40,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:41,236 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:41,540 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:42,557 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:42,849 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:45:42,849 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 with policy now +2025-01-07 22:45:42,870 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:42,871 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:42,871 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:42,871 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:43,059 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fuljjp7f-media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:43,378 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:45:43,562 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:43,562 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:43,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:43,656 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/7fbawmmv-media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:44,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:44,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:45,380 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:45,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:46,245 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:46,578 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:47,589 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:48,381 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:48,590 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:49,600 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:50,613 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:51,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:51,627 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:52,383 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:52,636 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:53,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:54,625 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:55,628 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:56,677 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:56,685 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:57,668 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:57,688 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:57,688 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:57,688 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:58,105 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:45:58,385 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:45:58,385 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:58,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:58,567 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:58,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:58,900 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:45:58,901 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:45:59,295 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:59,346 INFO wandb-upload_1:2243413 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp26yz91h4 +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table +2025-01-07 22:45:59,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:59,715 INFO SenderThread:2243413 [sender.py:send_request_log_artifact():1525] logged artifact run-opf0nkwg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU4NTY2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkwNDIzOA==', 'latestArtifact': None}} +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:45:59,715 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json with policy now +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,716 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send():379] send: telemetry +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,716 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,717 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,717 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,717 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,718 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,718 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,718 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,718 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,719 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,719 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,719 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,719 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,720 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,720 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,720 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,720 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,721 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,721 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,721 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,722 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:45:59,781 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:45:59,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:59,891 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/mac9pz84-media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:00,144 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/silpwdzp-media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:00,161 INFO wandb-upload_2:2243413 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq6r5uoya +2025-01-07 22:46:00,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:00,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval +2025-01-07 22:46:00,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:00,698 INFO SenderThread:2243413 [sender.py:send_request_log_artifact():1525] logged artifact run-opf0nkwg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU4NTY4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkwNDI0MA==', 'latestArtifact': None}} +2025-01-07 22:46:00,698 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:46:00,698 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json with policy now +2025-01-07 22:46:00,698 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:46:00,699 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:46:00,699 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:00,907 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/bicepv46-media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:00,965 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4mjpa5w5-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:01,156 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/57dh8lj3-media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:01,388 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:01,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:01,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:02,389 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:02,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:03,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:04,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:05,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:06,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:07,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:07,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:08,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:09,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:10,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:11,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:12,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:12,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:13,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:46:13,560 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:46:13,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:14,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:15,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:16,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:17,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:17,656 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:18,397 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:46:18,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:19,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:20,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:21,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:22,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:23,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:23,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:24,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:25,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:26,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:27,113 DEBUG SenderThread:2243413 [sender.py:send():379] send: exit +2025-01-07 22:46:27,113 INFO SenderThread:2243413 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 22:46:27,114 INFO SenderThread:2243413 [sender.py:send_exit():588] handling runtime: 1019 +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:send_exit():594] send defer +2025-01-07 22:46:27,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,115 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 22:46:27,115 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 1 +2025-01-07 22:46:27,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,115 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 22:46:27,116 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,116 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 22:46:27,116 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 2 +2025-01-07 22:46:27,116 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 22:46:27,116 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 22:46:27,116 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 22:46:27,117 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined network monitor +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 3 +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:46:27,171 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,171 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 4 +2025-01-07 22:46:27,171 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,171 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 22:46:27,402 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:28,402 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 22:46:33,700 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,700 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 22:46:33,700 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 5 +2025-01-07 22:46:33,701 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:33,701 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,701 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 22:46:33,701 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:46:33,702 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:46:33,702 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:33,702 DEBUG SenderThread:2243413 [sender.py:send():379] send: summary +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:33,706 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 6 +2025-01-07 22:46:33,706 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,706 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 22:46:33,706 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 7 +2025-01-07 22:46:33,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:33,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,707 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 22:46:33,707 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 22:46:34,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:34,405 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:37,029 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 8 +2025-01-07 22:46:37,029 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:37,029 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,030 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 22:46:37,030 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,030 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 22:46:37,030 INFO SenderThread:2243413 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 22:46:37,033 INFO SenderThread:2243413 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 22:46:37,035 INFO SenderThread:2243413 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 22:46:37,048 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 9 +2025-01-07 22:46:37,048 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,048 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 22:46:37,048 DEBUG SenderThread:2243413 [sender.py:send():379] send: artifact +2025-01-07 22:46:37,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 22:46:37,285 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 22:46:37,406 INFO SenderThread:2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml config.yaml +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json wandb-summary.json +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt requirements.txt +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json wandb-metadata.json +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log output.log +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch diff.patch +2025-01-07 22:46:37,408 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 22:46:37,413 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:46:37,413 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 10 +2025-01-07 22:46:37,415 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:37,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,415 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 22:46:37,416 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,416 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 22:46:37,416 INFO SenderThread:2243413 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:46:37,601 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:37,644 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt +2025-01-07 22:46:37,659 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:46:37,688 INFO wandb-upload_3:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:37,888 INFO Thread-11 (_thread_body):2243413 [sender.py:transition_state():614] send defer: 11 +2025-01-07 22:46:37,889 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,889 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 22:46:37,889 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,889 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 22:46:37,889 INFO SenderThread:2243413 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 12 +2025-01-07 22:46:37,892 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,892 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 22:46:37,892 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [file_stream.py:finish():601] file stream finish called +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [file_stream.py:finish():605] file stream finish is done +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 13 +2025-01-07 22:46:38,083 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:38,083 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 22:46:38,083 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 14 +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send():379] send: final +2025-01-07 22:46:38,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:38,084 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send():379] send: footer +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:38,084 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 22:46:38,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:38,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 22:46:38,085 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:38,085 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: server_info +2025-01-07 22:46:38,086 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:38,087 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 22:46:38,087 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 22:46:38,088 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:38,139 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:38,139 INFO MainThread:2243413 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 22:46:38,140 INFO MainThread:2243413 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 22:46:38,140 INFO MainThread:2243413 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 22:46:38,140 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 22:46:38,140 INFO HandlerThread:2243413 [handler.py:finish():882] shutting down handler +2025-01-07 22:46:39,087 INFO WriterThread:2243413 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [sender.py:finish():1615] shutting down sender +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [file_pusher.py:join():175] waiting for file pusher diff --git a/opf0nkwg/wandb/debug.log b/opf0nkwg/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f6047abe6c6f1f19eaea1c25738b18c6a1bd5031 --- /dev/null +++ b/opf0nkwg/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Configure stats pid to 2243246 +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/settings +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug.log +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug-internal.log +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:init():569] calling init triggers +2025-01-07 22:29:27,690 INFO MainThread:2243246 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg'} +2025-01-07 22:29:27,691 INFO MainThread:2243246 [wandb_init.py:init():619] starting backend +2025-01-07 22:29:27,691 INFO MainThread:2243246 [wandb_init.py:init():623] setting up manager +2025-01-07 22:29:27,695 INFO MainThread:2243246 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 22:29:27,696 INFO MainThread:2243246 [wandb_init.py:init():631] backend started and connected +2025-01-07 22:29:27,707 INFO MainThread:2243246 [wandb_init.py:init():720] updated telemetry +2025-01-07 22:29:27,713 INFO MainThread:2243246 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 22:29:28,003 INFO MainThread:2243246 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 22:29:28,077 INFO MainThread:2243246 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 22:29:28,077 INFO MainThread:2243246 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 22:29:28,408 INFO MainThread:2243246 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 22:29:28,408 INFO MainThread:2243246 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 22:29:28,409 INFO MainThread:2243246 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 22:29:28,409 INFO MainThread:2243246 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 22:29:28,410 INFO MainThread:2243246 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 22:29:28,410 INFO MainThread:2243246 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/tb, True +2025-01-07 22:29:32,802 INFO MainThread:2243246 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 22:46:39,141 WARNING MsgRouterThr:2243246 [router.py:message_loop():77] message_loop has been closed diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d2ec0d2027dd0e55769badebd01ff3c7fd53dd3 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736306967 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3f33d68d5f61ef69d5b4133841e12781c858f071 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch @@ -0,0 +1,30 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..0b377ae 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json new file mode 100644 index 0000000000000000000000000000000000000000..547b6bb2a0ca08768ff08056659bc0ea5df0d065 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.1667821706007775, 0, 0.08629500874210372, 0, 0.10209639378399515, 0, 0.10554929085514297]]} \ No newline at end of file diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json new file mode 100644 index 0000000000000000000000000000000000000000..050b9521ec30dc144b0f44f2a6d7a921d44ed182 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[41, 125, 0.328, 0.23784522717473777, 0.43069198728993907, 0.251689888586592, 0.4106370434343727, 0.2658199407628642, 0.39672438429871404, 0.2726878170378645, 0.38488362359322714]]} \ No newline at end of file diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08b03f344809c309de6c4d2eef96bca9ef7bf34b --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0416889a3e77e5f6820e3f79a7e68b35b8280549e19c319b2f836a6191ef8a3 +size 1212200 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..39a8e2c6e44cfd615d373f3cbef5ce779a558527 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119d64d622b2c3c1d8e6221298783f2fabf5f0bc6232a1dc0dda101bfb9ec6b8 +size 1295177 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5eb9f379a0c39892bb6ac256c94286e4b255921d --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15ed0619cc72d1cf6a653f631fdd6f13518cb33c0f5ea840a376904f2fcff7f +size 1216209 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..adb4b744531c57e5b07a15f9b171c3da3b5338a2 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d95a49b894bdfcd37b0c1c1669b8c6a141289459d04683071eb8ce38d88d77 +size 1203863 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3561c9594dae3dbddaaf254d76a34df8686bb2b3 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a7294f80db38ff7cb0178d8dff109458f6d59c9a5dc22995c9c82851c4a719 +size 1433965 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..219c0c53d1bd933fb7e768c0d4124ba70314f87e --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af06c0a0e058587490fe17759e07794abebc17c35366d1a1e6fb428d55f3fe4 +size 1448485 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e71cb82fc50db8b8b57f5c1deff94f3d0c4a7761 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de33546b235fc7477b2e4fccf32b2a5e8c12b207d50c7c65d2e8298fe04dcff6 +size 1475418 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d3ccb87cee91ca799d6db5ab8d01b4ca156161dc --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:991c282f4f03c1a3f74b839ca93393f883f7c889c8cdf9cb3ad22d513486d439 +size 1560712 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..37721556c6fd12aabe5487dccaa66911bf019abe --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf814c17374e1cd6bef0bc2afa3e1ef31988cdfe8e9f9de2860916e6dbb266c +size 1498337 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d9450eb119df72ba89b1039cec5992da8d8556e --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4987dfaab4d6ad84d0ce99e786a1dc0d0b67c7bf407fa710707d10a0c5614139 +size 1510751 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7e82206d064d00bcf460aef1e4e7466ff0695bef --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc271ae74232dc330acb4135e48ae2c481ea4f7e8499217eed6c7aabb081e22 +size 1187563 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e6b5e13255c10d430d260ff91315498995608124 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479971f5e97e50638f221f0625e45daf2e59bd333480512dfb081bb5fe7f0095 +size 1198438 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd59baf7f1241f39735a654aacf5e572d3ebf4da --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75d5fcf827136b5c43bf2d3469c641d32b0cfe6bf0d324a0c654a30c5b4fda8 +size 1159893 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74ceb5f08afaccb37a70cc6c8c1e3f82ce0df250 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4107d85ee8f655917ae7734e9fc72871bde4d98c147f3d5f8ae2145410178402 +size 1168378 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8a4b7e7a32d351eb84e685923aafcdc6eedc853d --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4af638a8cd5e4e155cbb4d7b2dcd013be4455d3a2146012c4a159d161cfdb5 +size 1265054 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..75ec30c982154e77cf312184ea607de48bc995dc --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa2adef9cffa563b3117db168e6fa5630369ec1bf420c8c03f2313c78c61347e +size 1213213 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87a6c522f8da27ac1a716a41f5161a421fb85bdb --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bdc6ce07f76b86e3067e1af08273a2e7a4399c69adc8e37d196402db65b2f3f +size 1217184 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..31707d85ce65f7100c43b13eabefa83e5fbbc71a --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94820970d3238dd1d30cad3b7fb5cdc5ff5db8a286232696e147349f717d41a +size 1282027 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65d96b2feeeed2bc179e736bef5e0017498f786a --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e5641ff60163a911937b6ca4cb77f87c02fc667fe16b75e90ec6e45c5951960 +size 1186965 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3152a7eb1216043b09a00230cae300744aba3761 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef5c54b11ebe40dec8b27aff851a0802e98ad3a05e55de1df66231e507a869a9 +size 1213538 diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..413cbcd4360753a1ec6efa9ceff7ff6efedde3fa --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log @@ -0,0 +1,389 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736306972.802824]: resetting environment +[INFO] [1736306972.806355]: cleared current plan +[INFO] [1736306972.807297]: moving to home +[INFO] [1736306973.809253]: reached home +[INFO] [1736306973.810550]: reset action history +[INFO] [1736306976.518878]: environment reset +Starting episode 0 +[INFO] [1736306976.521648]: resetting environment +[INFO] [1736306976.526224]: cleared current plan +[INFO] [1736306976.527663]: moving to home +[INFO] [1736306977.529429]: reached home +[INFO] [1736306977.530609]: reset action history +[INFO] [1736306980.239555]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736306983.938239]: resetting environment +[INFO] [1736306983.941094]: cleared current plan +[INFO] [1736306983.942275]: moving to home +[INFO] [1736306984.944852]: reached home +[INFO] [1736306984.947351]: reset action history +[INFO] [1736306987.656964]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/episode_rosbags/episode_0_2025-01-07-22-29-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2243246.log +started roslaunch server http://158.130.50.37:44279/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2243536] +process[print_text-2]: started with pid [2243537] +process[rosbag_record-3]: started with pid [2243538] +started bagging! +For topic gripper_width: timestamp difference is 117323569 for nearest: 1736306993572547606 - target: 1736306993455224037 at idx 70 +[INFO] [1736307014.737076]: Storing episode... +[print_text-2] killing on exit +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[INFO] [1736307015.545487]: Stored episode 1. +[INFO] [1736307015.545699]: Saving video... +[INFO] [1736307015.742694]: Video saved! +Starting episode 1 +[INFO] [1736307015.760190]: resetting environment +[INFO] [1736307015.766408]: cleared current plan +[INFO] [1736307015.766663]: moving to home +[INFO] [1736307019.169281]: reached home +[INFO] [1736307019.172739]: reset action history +[INFO] [1736307021.878478]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736307035.478579]: resetting environment +[INFO] [1736307035.482078]: cleared current plan +[INFO] [1736307035.482532]: moving to home +[INFO] [1736307036.484003]: reached home +[INFO] [1736307036.484780]: reset action history +[INFO] [1736307039.190261]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/episode_rosbags/episode_1_2025-01-07-22-30-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2243246.log +started roslaunch server http://158.130.50.37:37101/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2243650] +process[print_text-5]: started with pid [2243651] +process[rosbag_record-6]: started with pid [2243675] +started bagging! +For topic gripper_width: timestamp difference is 100648971 for nearest: 1736307058005880447 - target: 1736307057905231476 at idx 249 +For topic gripper_width: timestamp difference is 101944975 for nearest: 1736307058605929426 - target: 1736307058503984451 at idx 249 +For topic gripper_width: timestamp difference is -198424764 for nearest: 1736307064705880456 - target: 1736307064904305220 at idx 249 +[INFO] [1736307066.183040]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736307066.915098]: Stored episode 2. +[INFO] [1736307066.915888]: Saving video... +[INFO] [1736307067.158224]: Video saved! +Starting episode 2 +[INFO] [1736307067.201894]: resetting environment +[INFO] [1736307067.214268]: cleared current plan +[INFO] [1736307067.214607]: moving to home +[INFO] [1736307070.617639]: reached home +[INFO] [1736307070.619091]: reset action history +[INFO] [1736307073.403564]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736307077.966411]: resetting environment +[INFO] [1736307077.974948]: cleared current plan +[INFO] [1736307077.975642]: moving to home +[INFO] [1736307078.977258]: reached home +[INFO] [1736307078.977772]: reset action history +[INFO] [1736307081.686494]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/episode_rosbags/episode_2_2025-01-07-22-31-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2243246.log +started roslaunch server http://158.130.50.37:39559/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2243810] +process[print_text-8]: started with pid [2243811] +process[rosbag_record-9]: started with pid [2243812] +started bagging! +started bagging! +started bagging! +started bagging! +started bagging! +started bagging! +started bagging! +[WARN] [1736307110.200621]: Plan exhausted +started bagging! +started bagging! +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307152.601374]: Plan exhausted +[WARN] [1736307152.641617]: Plan exhausted +[WARN] [1736307152.671174]: Plan exhausted +[WARN] [1736307152.709480]: Plan exhausted +[WARN] [1736307152.748111]: Plan exhausted +[WARN] [1736307152.798892]: Plan exhausted +[WARN] [1736307152.830628]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307198.230609]: Plan exhausted +[WARN] [1736307198.271021]: Plan exhausted +[WARN] [1736307198.308567]: Plan exhausted +[WARN] [1736307198.347769]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307287.028671]: Plan exhausted +[WARN] [1736307287.069835]: Plan exhausted +[WARN] [1736307287.109610]: Plan exhausted +[WARN] [1736307287.148648]: Plan exhausted +[WARN] [1736307287.189677]: Plan exhausted +[WARN] [1736307287.227788]: Plan exhausted +[WARN] [1736307287.267807]: Plan exhausted +[WARN] [1736307287.307981]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +Starting episode 8ticence bound +Starting episode 8ticence bound +[WARN] [1736307333.948662]: Plan exhausted +[WARN] [1736307333.990309]: Plan exhausted +[WARN] [1736307334.032639]: Plan exhausted +[WARN] [1736307334.070036]: Plan exhausted +Starting episode 8ticence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307378.190089]: Plan exhausted +[WARN] [1736307378.227802]: Plan exhausted +[WARN] [1736307378.274989]: Plan exhausted +[WARN] [1736307378.310567]: Plan exhausted +[WARN] [1736307378.347817]: Plan exhausted +[WARN] [1736307378.388408]: Plan exhausted +[WARN] [1736307378.428024]: Plan exhausted +[WARN] [1736307378.468125]: Plan exhausted +[WARN] [1736307378.508119]: Plan exhausted +[WARN] [1736307378.547893]: Plan exhausted +[WARN] [1736307378.591325]: Plan exhausted +[WARN] [1736307378.632397]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307423.507825]: Plan exhausted +[WARN] [1736307423.556706]: Plan exhausted +[WARN] [1736307423.587944]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound +Starting episode 11icence bound +Starting episode 11icence bound +Starting episode 11icence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Starting episode 12idence bound +Starting episode 12idence bound +[WARN] [1736307516.228967]: Plan exhausted +[WARN] [1736307516.268239]: Plan exhausted +[WARN] [1736307516.319914]: Plan exhausted +[WARN] [1736307516.350749]: Plan exhausted +[WARN] [1736307516.387794]: Plan exhausted +[WARN] [1736307516.438804]: Plan exhausted +[WARN] [1736307516.468733]: Plan exhausted +Starting episode 12idence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +[WARN] [1736307561.356702]: Plan exhausted +[WARN] [1736307561.387905]: Plan exhausted +[WARN] [1736307561.429642]: Plan exhausted +[WARN] [1736307561.469875]: Plan exhausted +[WARN] [1736307561.507972]: Plan exhausted +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +[WARN] [1736307610.834664]: Plan exhausted +[WARN] [1736307610.871243]: Plan exhausted +[WARN] [1736307610.909718]: Plan exhausted +[WARN] [1736307610.948354]: Plan exhausted +[WARN] [1736307611.003063]: Plan exhausted +[WARN] [1736307611.041972]: Plan exhausted +[WARN] [1736307611.074256]: Plan exhausted + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +[WARN] [1736307681.669406]: Plan exhausted +[WARN] [1736307681.717838]: Plan exhausted +[WARN] [1736307681.753284]: Plan exhausted +[WARN] [1736307681.787804]: Plan exhausted +[WARN] [1736307681.836653]: Plan exhausted +[WARN] [1736307681.875267]: Plan exhausted + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +[WARN] [1736307733.547894]: Plan exhausted + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosversion: 1.16.0nce bound + * /rosversion: 1.16.0nce bound + * /rosversion: 1.16.0nce bound +[WARN] [1736307787.268152]: Plan exhausted +[WARN] [1736307787.331100]: Plan exhausted +[WARN] [1736307787.349817]: Plan exhausted +[WARN] [1736307787.388650]: Plan exhausted + * /rosversion: 1.16.0nce bound + * /rosversion: 1.16.0nce bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound +Starting episode 18icence bound +Starting episode 18icence bound +[WARN] [1736307842.714555]: Plan exhausted +Starting episode 18icence bound +Comuting lower confidence bound +Comuting lower confidence bound +Comuting lower confidence bound + * /rosdistro: noeticence bound + * /rosdistro: noeticence bound +[INFO] [1736307896.881347]: Video saved! +[INFO] [1736307896.881347]: Video saved! +[WARN] [1736307896.912586]: Plan exhausted +[WARN] [1736307896.969117]: Plan exhausted +[WARN] [1736307897.009686]: Plan exhausted +[WARN] [1736307897.047916]: Plan exhausted +[WARN] [1736307897.087138]: Plan exhausted +[WARN] [1736307897.136892]: Plan exhausted +[INFO] [1736307896.881347]: Video saved! +[INFO] [1736307896.881347]: Video saved! +Comuting lower confidence boundeo saved! +Comuting lower confidence boundeo saved! +Comuting lower confidence boundeo saved! +Comuting lower confidence boundeo saved! +Comuting lower confidence boundeo saved! +[WARN] [1736307942.389992]: Plan exhausted +[WARN] [1736307942.430380]: Plan exhausted +[WARN] [1736307942.475209]: Plan exhausted +[INFO] [1736307942.904763]: resetting environment +[INFO] [1736307942.904763]: resetting environment +[WARN] [1736307942.557307]: Plan exhausted +[WARN] [1736307942.590486]: Plan exhausted +[WARN] [1736307942.629280]: Plan exhausted +[WARN] [1736307942.670844]: Plan exhausted +[WARN] [1736307942.713837]: Plan exhausted +[WARN] [1736307942.748184]: Plan exhausted +[WARN] [1736307942.788576]: Plan exhausted +[WARN] [1736307942.827802]: Plan exhausted +[WARN] [1736307942.870347]: Plan exhausted +[WARN] [1736307942.924301]: Plan exhausted +[WARN] [1736307942.948208]: Plan exhausted +[INFO] [1736307942.904763]: resetting environment +Comuting lower confidence boundetting environment +Comuting upper confidence boundetting environment +Comuting upper confidence boundetting environment +Comuting upper confidence boundetting environment \ No newline at end of file diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a5204ed08a79a5dc568fe7c81a975158f0535773 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T03:29:28.100501", + "startedAt": "2025-01-08T03:29:27.682555", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.0784999999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.977, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4482.979, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 684.1397895812988 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6dd3d26372d42d20f69fa259de7290eb107d8d4f --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c0416889a3e77e5f6820e3f79a7e68b35b8280549e19c319b2f836a6191ef8a3", "size": 1212200, "path": "media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4"}, "global_step": 2657, "_timestamp": 1736307959.7013993, "_runtime": 992.005024433136, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "479971f5e97e50638f221f0625e45daf2e59bd333480512dfb081bb5fe7f0095", "size": 1198438, "path": "media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4"}, "eval/num_success": 0.0, "episode": 125.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.1667821705341339, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.08629500865936279, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.10209639370441437, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.10554929077625275, "eval/2_eval": {"_type": "video-file", "sha256": "e75d5fcf827136b5c43bf2d3469c641d32b0cfe6bf0d324a0c654a30c5b4fda8", "size": 1159893, "path": "media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4107d85ee8f655917ae7734e9fc72871bde4d98c147f3d5f8ae2145410178402", "size": 1168378, "path": "media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "da4af638a8cd5e4e155cbb4d7b2dcd013be4455d3a2146012c4a159d161cfdb5", "size": 1265054, "path": "media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "aa2adef9cffa563b3117db168e6fa5630369ec1bf420c8c03f2313c78c61347e", "size": 1213213, "path": "media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "1bdc6ce07f76b86e3067e1af08273a2e7a4399c69adc8e37d196402db65b2f3f", "size": 1217184, "path": "media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "b94820970d3238dd1d30cad3b7fb5cdc5ff5db8a286232696e147349f717d41a", "size": 1282027, "path": "media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "3e5641ff60163a911937b6ca4cb77f87c02fc667fe16b75e90ec6e45c5951960", "size": 1186965, "path": "media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "ef5c54b11ebe40dec8b27aff851a0802e98ad3a05e55de1df66231e507a869a9", "size": 1213538, "path": "media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "119d64d622b2c3c1d8e6221298783f2fabf5f0bc6232a1dc0dda101bfb9ec6b8", "size": 1295177, "path": "media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "a15ed0619cc72d1cf6a653f631fdd6f13518cb33c0f5ea840a376904f2fcff7f", "size": 1216209, "path": "media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "64d95a49b894bdfcd37b0c1c1669b8c6a141289459d04683071eb8ce38d88d77", "size": 1203863, "path": "media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "c0a7294f80db38ff7cb0178d8dff109458f6d59c9a5dc22995c9c82851c4a719", "size": 1433965, "path": "media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "3af06c0a0e058587490fe17759e07794abebc17c35366d1a1e6fb428d55f3fe4", "size": 1448485, "path": "media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "de33546b235fc7477b2e4fccf32b2a5e8c12b207d50c7c65d2e8298fe04dcff6", "size": 1475418, "path": "media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "991c282f4f03c1a3f74b839ca93393f883f7c889c8cdf9cb3ad22d513486d439", "size": 1560712, "path": "media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "abf814c17374e1cd6bef0bc2afa3e1ef31988cdfe8e9f9de2860916e6dbb266c", "size": 1498337, "path": "media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "4987dfaab4d6ad84d0ce99e786a1dc0d0b67c7bf407fa710707d10a0c5614139", "size": 1510751, "path": "media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "3cc271ae74232dc330acb4135e48ae2c481ea4f7e8499217eed6c7aabb081e22", "size": 1187563, "path": "media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d75e5a883a216df50b02ed3f42f452929bdd442d8755d2aed3f626dd15f0b604", "size": 333, "artifact_path": "wandb-client-artifact://qyadv4mw7k9o86w3eesrz890oe86gohwaihrmxnyts2lwcmahip9acwa4gw6hh576v5uhv6tsg2015o1k4fjr6jsf7q19q8zxvy8nnbpqryz8rdgmqc6zhd6iwzp6kid/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cgjns8ktqfiodrucw063k3fqaxtc9cetvvia3d90hrhevuwm806y2qy4lxi5jsuxcb5yobzv7zir1acghn1a6dkf1h6z2njmh7jtcpkm2u8jeejs918fdue8r645c2zo:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 41, "total_num_episodes": 125, "total_success_rate": 0.328, "total_success_rate_lb_0.01": 0.23784522717473777, "total_success_rate_ub_0.01": 0.43069198728993907, "total_success_rate_lb_0.025": 0.251689888586592, "total_success_rate_ub_0.025": 0.4106370434343727, "total_success_rate_lb_0.05": 0.2658199407628642, "total_success_rate_ub_0.05": 0.39672438429871404, "total_success_rate_lb_0.1": 0.2726878170378645, "total_success_rate_ub_0.1": 0.38488362359322714, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "fd9e313f895b1255cc987f8dde5245965c5869fa46bf87ad1c302bbd58ad0b1d", "size": 511, "artifact_path": "wandb-client-artifact://rzu3a0hohvm2prmoxolchx2ry5o24iggyu7ciyukw2lhdcxq5x8kfdw8i6itnsusdiigp74fzb7n5n56vnrs3ypwcjng931910nydowbmnluncoo3p57m7flt7cjcaao/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cx48nh33oel97cpfio54849grd8hbtrb1wgaram8cfjm49o7paqdqqmgmheftu50ni1cj627ewvryrs1cr4fv8hci6dc0zx3g1evgcp0vfb7qewun1nvq4hv7tptqz5l:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1019}, "eval/total_success_rate": 0.328000009059906, "eval/total_success_rate_lb_0.01": 0.23784522712230682, "eval/total_success_rate_ub_0.01": 0.43069198727607727, "eval/total_success_rate_lb_0.025": 0.2516898810863495, "eval/total_success_rate_ub_0.025": 0.4106370508670807, "eval/total_success_rate_lb_0.05": 0.2658199369907379, "eval/total_success_rate_ub_0.05": 0.3967243731021881, "eval/total_success_rate_lb_0.1": 0.2726878225803375, "eval/total_success_rate_ub_0.1": 0.3848836123943329} \ No newline at end of file diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug-internal.log b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..99d23c3410ef2f63a4276be3b67f696e38c0ed72 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug-internal.log @@ -0,0 +1,2184 @@ +2025-01-07 22:29:27,696 INFO StreamThr :2243413 [internal.py:wandb_internal():85] W&B internal server running at pid: 2243413, started at: 2025-01-07 22:29:27.696166 +2025-01-07 22:29:27,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status +2025-01-07 22:29:27,707 INFO WriterThread:2243413 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb +2025-01-07 22:29:27,707 DEBUG SenderThread:2243413 [sender.py:send():379] send: header +2025-01-07 22:29:27,716 DEBUG SenderThread:2243413 [sender.py:send():379] send: run +2025-01-07 22:29:27,985 INFO SenderThread:2243413 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files +2025-01-07 22:29:27,985 INFO SenderThread:2243413 [sender.py:_start_run_threads():1188] run started: opf0nkwg with start time 1736306967.696375 +2025-01-07 22:29:28,004 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 22:29:28,004 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: check_version +2025-01-07 22:29:28,082 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 22:29:28,087 DEBUG HandlerThread:2243413 [system_info.py:__init__():26] System info init +2025-01-07 22:29:28,087 DEBUG HandlerThread:2243413 [system_info.py:__init__():41] System info init done +2025-01-07 22:29:28,087 INFO HandlerThread:2243413 [system_monitor.py:start():194] Starting system monitor +2025-01-07 22:29:28,087 INFO SystemMonitor:2243413 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 22:29:28,087 INFO HandlerThread:2243413 [system_monitor.py:probe():214] Collecting system info +2025-01-07 22:29:28,088 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 22:29:28,089 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started disk monitoring +2025-01-07 22:29:28,089 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 22:29:28,090 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started memory monitoring +2025-01-07 22:29:28,090 INFO SystemMonitor:2243413 [interfaces.py:start():188] Started network monitoring +2025-01-07 22:29:28,100 DEBUG HandlerThread:2243413 [system_info.py:probe():152] Probing system +2025-01-07 22:29:28,101 DEBUG HandlerThread:2243413 [system_info.py:_probe_git():137] Probing git +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:_probe_git():145] Probing git done +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:probe():200] Probing system done +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T03:29:28.100501', 'startedAt': '2025-01-08T03:29:27.682555', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.0784999999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.977, 'min': 800.0, 'max': 4700.0}, {'current': 4482.979, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 684.1397895812988}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 22:29:28,106 INFO HandlerThread:2243413 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 22:29:28,106 INFO HandlerThread:2243413 [system_monitor.py:probe():227] Publishing system info +2025-01-07 22:29:28,106 DEBUG HandlerThread:2243413 [system_info.py:_save_code():44] Saving code +2025-01-07 22:29:28,108 DEBUG HandlerThread:2243413 [system_info.py:_save_code():67] Saving code done +2025-01-07 22:29:28,108 DEBUG HandlerThread:2243413 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 22:29:28,170 DEBUG HandlerThread:2243413 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 22:29:28,170 INFO HandlerThread:2243413 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 22:29:28,173 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:29:28,173 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 22:29:28,173 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 22:29:28,174 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 22:29:28,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 22:29:28,407 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: python_packages +2025-01-07 22:29:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:28,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:28,411 INFO HandlerThread:2243413 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/tb" + save: true +} +control { + relay_id: "092b5698c5204f738cc04f390ab07652" +} +_info { + stream_id: "opf0nkwg" +} + +2025-01-07 22:29:28,411 DEBUG HandlerThread:2243413 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 22:29:28,414 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/mlfwd2eb-diff.patch +2025-01-07 22:29:28,436 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9wprl8i7-wandb-metadata.json +2025-01-07 22:29:28,442 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zxwa01co-code/FISH/eval_robot.py +2025-01-07 22:29:28,506 DEBUG SenderThread:2243413 [sender.py:send():379] send: telemetry +2025-01-07 22:29:28,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: tbrecord +2025-01-07 22:29:28,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:29:28,507 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 with policy live +2025-01-07 22:29:28,687 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/raltsj4z-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py +2025-01-07 22:29:28,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH +2025-01-07 22:29:28,988 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code +2025-01-07 22:29:29,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:30,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:30,986 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:31,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:32,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:32,802 DEBUG SenderThread:2243413 [sender.py:send():379] send: config +2025-01-07 22:29:32,802 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:33,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:34,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:34,987 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:35,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:36,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:37,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:38,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:38,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:38,988 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:39,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:40,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:41,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:42,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:42,989 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:43,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:43,407 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:43,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:43,491 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:44,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:44,990 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:45,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:46,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:46,991 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:47,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:48,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:48,991 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:49,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:49,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:50,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:51,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:52,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:53,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:54,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:54,621 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:29:54,993 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:29:55,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:56,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:57,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:58,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:29:58,408 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:29:58,449 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:29:59,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:00,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:00,571 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:00,994 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:30:01,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:02,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:03,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:04,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:05,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:06,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:06,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:07,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:08,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:09,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:10,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:11,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:12,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:12,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:13,408 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:13,408 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:13,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:14,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:15,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:15,759 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:30:15,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:30:15,759 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 with policy now +2025-01-07 22:30:15,760 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:30:15,760 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:30:15,761 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:30:15,980 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/b2g0vvz0-media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:15,998 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:30:15,998 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos +2025-01-07 22:30:15,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media +2025-01-07 22:30:16,302 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/gsa7cvl6-media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:30:16,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:16,999 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:17,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:17,767 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:18,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:19,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:20,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:21,000 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:21,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:22,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:22,925 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:23,001 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:23,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:24,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:25,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:26,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:27,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:28,090 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 22:30:28,091 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:30:28,092 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:28,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:28,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:28,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:29,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:30,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:31,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:32,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:33,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:33,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:34,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:35,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:36,005 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:30:36,251 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/20er5t1z-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:30:36,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:37,005 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:37,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:38,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:39,006 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:39,190 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:39,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:40,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:41,006 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:41,429 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:42,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:43,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:43,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:43,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:44,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:44,537 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:45,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:46,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:47,430 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:48,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:49,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:50,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:50,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:51,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:52,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:53,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:54,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:55,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:30:55,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:56,431 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:57,432 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:58,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:30:58,409 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:30:58,409 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:30:58,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:30:59,012 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:30:59,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:00,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:00,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:01,013 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:01,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:02,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:03,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:04,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:05,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:06,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:06,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:07,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:07,197 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:31:07,197 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 with policy now +2025-01-07 22:31:07,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:31:07,200 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:07,200 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:07,200 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:07,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:07,450 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/60aix6kp-media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:31:08,284 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fk371zr3-media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:31:08,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:09,015 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:09,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:10,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:11,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:11,661 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:12,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:13,017 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:13,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:13,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:13,445 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:14,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:15,017 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:15,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:16,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:17,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:17,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:18,018 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:31:18,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:18,822 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:18,822 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:18,822 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:19,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:19,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:19,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:20,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:21,019 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:21,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:22,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:23,020 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:23,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:23,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:24,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:25,021 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:25,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:26,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:27,440 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:28,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:31:28,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:28,410 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:28,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:28,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:29,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:30,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:31,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:32,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:33,023 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:33,441 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:34,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:34,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:35,024 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:35,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:36,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:37,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:38,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:39,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:40,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:40,442 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:41,026 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:41,444 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:42,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:43,029 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:43,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:43,410 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:43,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:44,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:45,445 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:45,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:46,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:47,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:48,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:49,448 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:50,196 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:31:50,196 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 with policy now +2025-01-07 22:31:50,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:31:50,200 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:31:50,200 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:31:50,200 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:31:50,440 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/knmflj8v-media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:50,455 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:51,032 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:31:51,226 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:51,295 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/a38l9q0o-media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:31:51,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:52,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:53,033 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:53,463 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:54,461 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:55,033 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:55,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:56,472 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:56,512 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:31:57,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:58,092 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:31:58,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:31:58,412 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:31:58,467 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:31:59,034 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:31:59,472 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:00,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:00,656 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:00,656 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:00,656 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:01,035 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:01,035 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:01,286 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/d6rnbtvx-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:01,470 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:01,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:02,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:03,036 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:03,471 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:04,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:05,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:06,468 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:06,750 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:07,038 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:07,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:08,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:09,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:10,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:11,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:12,417 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:12,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:13,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:13,411 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:13,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:14,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:15,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:16,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:17,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:17,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:18,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:19,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:20,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:21,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:22,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:23,286 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:23,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:24,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:25,052 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:25,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:26,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:27,053 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:27,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:28,093 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:32:28,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:28,412 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:28,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:28,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:29,054 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:29,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:30,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:31,054 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:31,497 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:32,484 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:32,827 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:32:32,827 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 with policy now +2025-01-07 22:32:32,831 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:32:32,831 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:32,831 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:32,831 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:33,055 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:32:33,059 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zknf0zt1-media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,328 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/j2cl0tip-media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:32:33,480 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:34,436 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:34,482 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:35,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:35,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:36,493 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:37,056 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:37,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:38,491 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:39,503 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:40,021 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:40,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:41,058 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:41,496 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:42,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:43,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:43,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:43,503 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:44,541 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:44,710 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:32:44,710 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:32:44,710 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:32:45,059 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:32:45,059 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:32:45,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:45,562 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:46,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:47,060 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:47,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:48,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:49,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:50,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:51,061 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:32:51,205 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:51,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:52,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:53,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:54,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:55,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:56,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:32:56,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:57,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:58,093 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:32:58,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:32:58,413 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:32:58,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:32:59,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:00,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:01,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:01,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:02,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:03,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:04,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:05,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:06,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:07,413 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:07,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:08,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:09,067 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:09,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:10,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:11,068 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:11,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:12,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:13,257 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:13,412 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:13,413 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:13,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:14,508 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:15,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:16,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:17,514 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:18,274 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:18,335 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:33:18,335 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 with policy now +2025-01-07 22:33:18,343 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:33:18,347 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:33:18,347 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:33:18,347 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:33:18,512 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:18,584 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/raznu5ux-media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,071 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:33:19,071 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:19,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:33:19,342 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/to5pswpt-media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:33:19,514 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:20,518 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:21,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:21,531 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:22,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:23,072 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:23,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:23,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:24,530 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:25,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:26,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:27,074 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:27,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:28,094 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:33:28,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:28,421 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:28,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:28,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:29,539 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:30,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:31,075 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:33:31,324 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/kz9w931i-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:33:31,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:31,766 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:33:31,766 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:33:31,767 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:33:32,076 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:33:32,537 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:33,076 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:33,543 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:34,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:34,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:35,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:36,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:37,077 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:37,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:38,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:39,078 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:33:39,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:40,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:40,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:41,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:42,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:43,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:43,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:43,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:44,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:45,506 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:45,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:46,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:47,546 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:48,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:49,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:50,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:51,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:51,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:52,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:53,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:54,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:55,548 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:56,426 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:33:56,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:57,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:58,094 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:33:58,414 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:33:58,414 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:33:58,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:33:59,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:00,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:01,086 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:01,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:02,176 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:02,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:03,550 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:04,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:04,985 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:34:04,985 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:34:04,985 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 with policy now +2025-01-07 22:34:04,986 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:04,986 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:04,986 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,088 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:34:05,279 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/ytldlvrg-media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,550 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/86ht2tnr-media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:34:05,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:06,571 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:07,089 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:07,558 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:08,026 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:08,563 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:09,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:10,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:11,090 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:11,556 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:12,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:13,091 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:13,421 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:13,423 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:13,423 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:13,579 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:14,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:14,820 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:14,820 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:14,821 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:15,092 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:15,092 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:15,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:16,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:17,093 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:17,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:18,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:18,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:19,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:20,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:21,094 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:21,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:22,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:23,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:24,309 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:24,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:25,096 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:25,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:26,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:27,096 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:27,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:28,095 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:34:28,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:28,415 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:28,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:29,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:29,575 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:30,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:31,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:32,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:33,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:34,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:35,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:35,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:36,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:37,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:38,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:39,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:40,433 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:40,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:41,570 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:42,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:43,416 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:43,416 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:43,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:44,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:45,103 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:45,572 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:45,703 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:46,594 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:47,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:47,271 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:34:47,271 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 with policy now +2025-01-07 22:34:47,280 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:34:47,281 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:47,281 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:47,281 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:47,492 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fu3ls3bi-media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:47,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:48,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:34:48,376 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/qbs2cwev-media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:34:48,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:49,104 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:49,578 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:50,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:50,743 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:51,583 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:52,599 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:53,106 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:53,595 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:54,595 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:55,107 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:55,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:56,598 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:56,677 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:34:57,597 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:58,095 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:34:58,422 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:34:58,422 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:34:58,600 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:34:58,883 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:34:58,883 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:34:58,883 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:34:59,108 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:34:59,363 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/cz54zzq4-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:34:59,599 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:00,617 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:01,109 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:01,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:02,362 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:02,596 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:03,110 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:03,617 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:04,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:05,110 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:05,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:06,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:07,111 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:07,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:07,897 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:08,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:09,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:10,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:11,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:12,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:13,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:13,418 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:13,502 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:13,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:14,601 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:15,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:16,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:17,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:18,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:19,115 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:19,434 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:19,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:20,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:21,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:22,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:23,117 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:23,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:24,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:24,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:25,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:26,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:27,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:28,096 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:35:28,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:28,418 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:28,604 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:29,501 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:29,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:30,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:31,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:32,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:33,633 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:34,031 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:35:34,031 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 with policy now +2025-01-07 22:35:34,046 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:35:34,046 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:35:34,046 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:35:34,047 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:35:34,273 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/bw7cmt2v-media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,529 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4b7pq9rq-media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:35:34,612 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:35,093 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:35,121 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:35,625 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:36,641 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:37,122 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:37,644 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:38,636 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:39,123 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:39,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:40,147 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:40,634 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:41,123 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:41,641 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:42,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:43,424 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:43,424 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:43,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:44,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:45,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:45,645 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:46,648 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:46,942 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:35:46,942 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:35:46,942 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:35:47,125 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:35:47,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:35:47,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:47,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:48,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:49,126 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:49,663 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:50,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:51,127 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:35:51,364 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:51,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:52,650 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:53,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:54,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:55,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:56,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:35:56,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:57,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:58,096 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:35:58,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:35:58,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:35:58,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:35:59,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:00,651 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:01,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:01,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:02,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:03,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:04,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:05,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:06,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:07,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:07,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:08,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:09,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:10,652 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:11,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:12,438 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:12,653 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:13,418 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:13,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:13,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:14,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:15,658 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:16,659 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:17,660 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:17,877 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:18,577 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:36:18,577 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 with policy now +2025-01-07 22:36:18,580 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:36:18,580 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:36:18,581 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:36:18,581 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:36:18,684 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:18,812 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/04bg8kel-media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,137 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:36:19,137 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:19,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:36:19,411 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/wil3hctr-media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:36:19,682 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:20,671 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:21,138 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:21,676 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:22,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:23,139 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:23,451 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:23,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:24,713 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:25,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:26,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:27,140 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:27,678 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:28,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:36:28,423 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:28,423 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:28,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:28,684 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:29,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:30,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:30,995 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:36:30,995 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:36:30,996 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:36:31,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:36:31,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:36:31,471 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/6hri6jy4-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:36:31,691 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:32,704 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:33,142 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:33,719 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:34,461 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:34,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:35,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:36,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:37,144 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:37,686 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:38,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:39,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:40,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:40,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:41,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:42,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:43,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:43,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:43,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:44,687 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:45,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:45,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:46,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:47,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:48,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:49,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:50,688 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:51,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:51,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:52,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:53,152 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:36:53,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:54,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:55,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:56,453 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:36:56,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:57,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:58,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:36:58,419 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:36:58,419 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:36:58,689 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:36:59,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:00,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:01,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:01,690 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:02,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:03,155 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:03,553 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:37:03,554 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 with policy now +2025-01-07 22:37:03,557 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:37:03,558 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:03,558 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:03,558 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:03,722 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:03,776 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/10063bj9-media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:37:04,426 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/xo1c04lk-media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:37:04,711 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:05,156 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:05,697 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:06,622 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:06,734 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:07,728 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:08,726 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:09,158 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:09,729 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:10,744 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:11,159 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:11,756 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:12,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:12,732 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:13,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:13,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:13,737 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:14,773 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:15,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:16,049 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:16,049 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:16,049 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:16,160 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:37:16,161 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:16,753 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:17,161 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:17,765 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:17,977 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:18,771 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:19,162 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:19,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:20,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:21,163 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:21,766 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:22,774 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:23,038 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:23,778 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:24,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:25,164 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:25,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:26,758 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:27,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:28,097 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:37:28,098 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:28,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:28,446 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:28,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:29,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:30,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:31,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:32,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:33,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:33,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:34,759 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:35,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:36,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:37,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:38,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:38,760 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:39,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:40,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:41,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:42,761 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:43,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:43,446 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:43,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:43,762 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:44,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:45,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:46,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:47,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:48,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:49,462 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:49,763 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:50,764 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:51,768 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:52,507 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:37:52,507 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 with policy now +2025-01-07 22:37:52,511 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:37:52,511 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:37:52,511 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:37:52,512 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:37:52,757 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/kazi6hf6-media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:52,774 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:53,176 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:53,177 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:37:53,446 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/oparcluk-media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:37:53,795 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:54,485 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:37:54,777 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:55,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:56,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:57,178 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:37:57,802 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:58,098 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:37:58,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:37:58,473 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:37:58,788 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:37:59,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:00,009 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:00,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:01,179 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:01,776 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:02,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:03,791 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:04,110 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:04,110 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:04,111 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:04,180 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:04,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:04,454 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/jemmtlhf-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:04,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:05,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:05,798 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:05,829 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:06,804 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:07,181 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:07,800 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:08,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:09,182 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:09,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:10,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:11,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:11,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:12,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:13,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:13,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:13,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:14,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:15,185 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:15,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:16,475 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:16,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:17,185 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:17,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:18,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:19,783 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:20,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:21,187 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:21,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:22,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:22,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:23,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:24,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:25,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:26,784 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:27,189 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:27,282 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:27,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:28,098 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:38:28,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:28,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:28,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:29,190 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:29,785 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:30,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:31,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:32,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:32,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:33,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:34,786 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:35,792 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:36,390 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:38:36,391 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 with policy now +2025-01-07 22:38:36,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:38:36,411 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:36,411 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:36,412 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:36,610 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/w1v9s7ig-media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:36,793 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:37,193 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:37,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:38:37,464 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9b6tmmo5-media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:38:37,811 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:38,490 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:38,834 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:39,194 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:39,812 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:40,815 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:41,195 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:41,801 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:42,804 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:43,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:43,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:43,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:43,811 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:44,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:45,197 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:45,817 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:46,845 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:47,823 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:48,701 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:48,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:49,167 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:38:49,168 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:38:49,168 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:38:49,198 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:38:49,198 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:38:49,851 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:50,841 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:51,199 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:51,846 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:52,850 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:53,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:54,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:54,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:55,201 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:38:55,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:56,836 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:57,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:58,099 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:38:58,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:38:58,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:38:58,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:38:59,568 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:38:59,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:00,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:01,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:02,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:03,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:04,837 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:05,483 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:05,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:06,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:07,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:08,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:09,207 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:09,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:10,838 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:11,208 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:11,257 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:11,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:12,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:13,209 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:13,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:13,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:13,839 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:14,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:15,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:16,696 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:16,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:17,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:18,840 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:19,841 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:20,862 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:21,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:21,441 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:39:21,441 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 with policy now +2025-01-07 22:39:21,449 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:39:21,450 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:39:21,450 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:39:21,450 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:39:21,667 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/rcm81113-media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:21,887 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:22,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:39:22,496 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fwy80wuk-media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:39:22,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:22,883 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:23,213 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:23,862 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:24,879 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:25,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:26,885 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:27,215 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:27,919 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:27,993 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:28,099 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:39:28,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:28,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:28,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:29,215 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:29,891 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:30,910 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:31,907 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:32,941 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:33,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:39:33,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:33,222 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:39:33,222 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:39:33,222 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:33,222 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:39:33,962 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:34,217 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:39:34,951 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:35,218 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:35,973 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:36,945 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:37,218 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:37,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:38,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:38,956 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:39,219 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:39,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:40,965 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:41,955 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:42,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:43,221 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:43,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:43,447 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:43,602 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:43,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:44,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:45,222 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:45,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:46,949 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:47,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:48,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:49,224 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:49,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:49,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:50,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:51,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:52,950 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:53,226 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:53,951 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:54,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:54,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:55,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:56,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:57,228 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:39:57,952 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:58,100 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:39:58,450 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:39:58,450 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:39:58,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:39:59,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:39:59,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:00,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:01,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:02,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:03,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:04,953 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:05,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:05,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:06,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:07,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:08,954 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:09,977 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:10,845 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:10,982 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:11,049 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:40:11,049 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:40:11,050 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 with policy now +2025-01-07 22:40:11,050 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:40:11,050 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:40:11,050 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:40:11,233 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:40:11,288 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/yhc8yu3e-media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,562 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/253o2wip-media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:40:11,975 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:13,002 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:13,234 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:13,494 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:13,494 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:14,001 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:15,003 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:15,235 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:15,999 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:16,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:17,003 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:17,983 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:19,011 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:19,237 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:20,005 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:21,002 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:22,034 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:22,507 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:23,007 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:24,015 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:25,014 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:26,005 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:27,036 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:28,011 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:28,100 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:40:28,101 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:28,498 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:28,498 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:29,043 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:30,012 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:31,016 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:32,016 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:33,032 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:33,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:34,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:35,054 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:36,027 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:37,054 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:38,036 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:39,066 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:39,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:40,060 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:41,063 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:42,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:42,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:40:42,306 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:40:42,306 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:40:42,308 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:40:42,506 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/09wds7yj-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:40:43,058 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:43,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:40:43,246 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:43,486 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:43,486 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:44,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:44,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:45,062 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:46,057 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:47,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:47,248 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:48,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:49,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:50,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:50,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:51,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:52,106 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:53,117 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:54,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:55,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:55,251 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:40:56,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:56,157 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:40:57,078 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:58,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:40:58,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:40:58,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:40:58,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:40:59,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:00,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:01,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:01,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:02,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:03,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:04,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:05,079 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:06,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:07,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:07,516 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:08,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:09,080 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:10,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:11,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:12,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:12,516 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:13,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:13,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:13,477 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:14,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:15,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:16,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:17,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:17,575 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:18,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:19,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:20,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:21,096 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:21,265 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:21,817 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:41:21,817 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 with policy now +2025-01-07 22:41:21,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:41:21,822 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:41:21,822 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:41:21,822 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:41:22,037 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/lcdgfjst-media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,088 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:22,265 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:41:22,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:41:22,533 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/vrjpjmsf-media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:41:22,915 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:23,117 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:23,266 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:24,123 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:25,108 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:26,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:27,109 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:27,267 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:28,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:41:28,102 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:28,127 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:28,476 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:28,476 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:29,121 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:29,268 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:30,122 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:31,112 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:32,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:33,137 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:33,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:34,124 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:34,270 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:41:34,370 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:41:34,370 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:41:34,370 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:41:35,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:35,271 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:41:35,271 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:36,170 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:37,145 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:38,132 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:38,529 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:39,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:39,272 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:40,141 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:41,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:42,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:43,177 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:43,489 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:43,489 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:43,631 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:44,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:45,148 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:46,157 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:47,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:47,275 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:48,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:49,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:49,519 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:50,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:51,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:52,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:53,149 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:54,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:54,519 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:41:55,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:56,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:57,150 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:58,101 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:41:58,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:58,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:41:58,477 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:41:59,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:41:59,280 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:41:59,633 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:00,151 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:01,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:02,152 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:03,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:04,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:05,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:05,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:06,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:07,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:08,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:09,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:10,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:10,520 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:11,153 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:12,154 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:13,156 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:13,286 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:13,485 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:42:13,485 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 with policy now +2025-01-07 22:42:13,504 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:13,504 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:13,522 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:42:13,606 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:42:13,606 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:42:13,607 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:42:13,716 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/zvk9rtev-media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:14,207 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:14,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:42:14,560 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/p1z3uba1-media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:42:15,183 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:15,287 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:15,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:16,172 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:17,181 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:18,187 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:19,183 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:19,289 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:20,221 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:21,212 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:21,281 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:21,290 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:22,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:23,229 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:24,234 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:25,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:25,429 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:42:25,429 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:42:25,430 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:42:26,239 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:26,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:42:26,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:42:26,521 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:27,256 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:27,292 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:28,102 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:42:28,235 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:28,536 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:28,536 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:29,240 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:30,243 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:31,253 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:31,294 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:31,833 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:32,282 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:33,252 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:34,261 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:35,260 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:36,255 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:37,262 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:37,587 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:38,259 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:39,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:39,297 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:40,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:41,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:41,298 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:42:42,247 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:43,199 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:43,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:43,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:43,505 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:44,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:45,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:46,248 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:47,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:48,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:48,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:49,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:50,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:51,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:52,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:53,249 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:54,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:54,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:42:55,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:56,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:57,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:58,102 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:42:58,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:58,509 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:42:58,509 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:42:59,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:42:59,619 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:00,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:01,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:02,250 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:03,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:04,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:05,251 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:05,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:06,274 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:07,273 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:07,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:07,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:43:07,340 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:43:07,340 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 with policy now +2025-01-07 22:43:07,381 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:43:07,382 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:43:07,382 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:43:07,382 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:43:07,565 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/r6t0t2ft-media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:08,260 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:08,309 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:43:08,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:08,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:08,584 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/jagjpk1y-media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:43:09,278 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:09,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:10,305 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:10,310 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:10,597 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:11,270 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:12,298 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:12,311 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:13,288 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:13,527 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:13,528 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:14,271 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:15,296 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:16,102 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:16,312 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:16,313 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:17,297 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:18,309 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:19,321 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:20,294 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:20,314 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:43:20,500 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:43:20,500 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:43:20,501 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:43:20,576 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/oabo0or1-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:43:21,185 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:21,297 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:21,315 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:43:22,302 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:22,315 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:23,318 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:24,316 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:24,319 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:25,331 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:26,307 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:26,317 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:26,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:27,319 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:28,110 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:43:28,331 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:28,529 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:28,529 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:29,355 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:30,370 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:31,361 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:31,672 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:32,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:33,352 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:34,360 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:35,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:36,324 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:43:36,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:37,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:37,622 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:38,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:39,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:40,350 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:41,351 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:42,355 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:43,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:43,515 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:43,515 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:43,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:44,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:45,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:46,354 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:47,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:48,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:49,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:49,582 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:50,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:51,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:52,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:53,356 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:54,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:55,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:55,582 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:43:56,357 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:57,359 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:58,103 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:43:58,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:43:58,517 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:43:58,517 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:43:59,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:00,358 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:00,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:01,385 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:02,368 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:02,607 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:44:02,607 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 with policy now +2025-01-07 22:44:02,632 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:44:02,632 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:02,633 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:02,634 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:02,843 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/9t2sw8oy-media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:03,335 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:03,335 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:03,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:44:03,407 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:03,614 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4l9js92x-media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:44:04,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:04,395 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:05,336 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:05,406 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:06,182 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:06,386 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:07,401 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:08,337 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:08,394 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:09,410 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:10,422 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:11,446 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:11,505 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:12,339 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:12,429 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:13,427 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:13,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:13,545 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:14,411 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:15,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:16,341 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:44:16,439 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:16,568 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:16,569 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:16,569 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:16,569 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:17,341 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:17,437 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:18,342 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:18,435 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:19,451 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:20,343 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:20,447 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:21,464 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:21,605 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:22,344 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:22,464 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:23,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:24,467 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:25,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:26,499 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:26,638 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:27,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:28,104 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:44:28,456 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:28,530 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:28,531 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:29,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:30,347 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:30,457 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:31,466 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:31,787 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:32,466 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:33,469 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:34,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:35,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:36,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:37,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:37,614 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:38,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:39,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:40,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:41,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:42,473 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:42,614 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:43,474 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:43,524 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:43,524 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:44,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:45,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:46,354 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:46,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:47,477 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:47,765 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:48,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:49,478 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:50,487 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:51,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:52,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:53,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:53,615 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:54,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:55,479 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:56,486 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:57,048 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:44:57,048 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 with policy now +2025-01-07 22:44:57,052 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:44:57,053 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:44:57,053 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:44:57,053 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:44:57,298 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/vvmyr64k-media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:57,358 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:44:57,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:57,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:44:57,484 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:57,634 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/ubqtz0wg-media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:44:58,104 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:44:58,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:58,541 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:44:58,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:44:58,552 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:44:58,637 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:44:59,359 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:44:59,542 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:00,360 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:00,573 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:01,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:02,361 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:02,573 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:03,539 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:03,637 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:04,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:05,545 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:06,362 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:06,528 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:07,549 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:08,577 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:08,656 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:09,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:09,632 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:09,632 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:09,633 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:10,364 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:45:10,364 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:10,547 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:11,561 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:12,365 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:12,552 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:13,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:13,558 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:13,558 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:13,932 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:14,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:15,531 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:16,367 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:16,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:17,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:18,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:19,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:19,624 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:20,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:21,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:22,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:23,532 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:24,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:25,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:25,623 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:26,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:27,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:28,105 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:45:28,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:28,540 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:28,540 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:29,533 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:30,373 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:30,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:30,632 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:31,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:32,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:33,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:34,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:35,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:36,081 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:36,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:37,534 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:38,376 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:38,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:39,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:40,535 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:41,236 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:41,540 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:42,557 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:42,849 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:45:42,849 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 with policy now +2025-01-07 22:45:42,870 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:42,871 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:42,871 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:42,871 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:43,059 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/fuljjp7f-media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:43,378 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:43,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval +2025-01-07 22:45:43,562 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:43,562 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:43,574 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:43,656 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/7fbawmmv-media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:45:44,379 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:44,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:45,380 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:45,576 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:46,245 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:46,578 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:47,589 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:48,381 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:48,590 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:49,600 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:50,613 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:51,523 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:51,627 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:52,383 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:45:52,636 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:53,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:54,625 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:55,628 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:56,677 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:45:56,685 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:57,668 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:57,688 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:57,688 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:57,688 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:58,105 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:45:58,385 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:45:58,385 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:45:58,567 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:45:58,567 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:45:58,657 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:58,900 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:45:58,901 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:45:59,295 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:59,346 INFO wandb-upload_1:2243413 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp26yz91h4 +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval +2025-01-07 22:45:59,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table +2025-01-07 22:45:59,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:45:59,715 INFO SenderThread:2243413 [sender.py:send_request_log_artifact():1525] logged artifact run-opf0nkwg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU4NTY2Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkwNDIzOA==', 'latestArtifact': None}} +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:45:59,715 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json with policy now +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:45:59,715 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,716 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send():379] send: telemetry +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,716 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,716 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,717 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,717 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,717 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,718 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,718 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,718 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,718 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,719 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,719 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,719 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,719 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,720 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,720 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,720 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,720 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,721 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,721 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,721 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:45:59,722 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:45:59,780 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 22:45:59,781 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 22:45:59,821 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:45:59,891 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/mac9pz84-media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:00,144 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/silpwdzp-media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:00,161 INFO wandb-upload_2:2243413 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq6r5uoya +2025-01-07 22:46:00,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:00,386 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:00,387 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval +2025-01-07 22:46:00,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:00,698 INFO SenderThread:2243413 [sender.py:send_request_log_artifact():1525] logged artifact run-opf0nkwg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjU4NTY4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkwNDI0MA==', 'latestArtifact': None}} +2025-01-07 22:46:00,698 DEBUG SenderThread:2243413 [sender.py:send():379] send: files +2025-01-07 22:46:00,698 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json with policy now +2025-01-07 22:46:00,698 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:46:00,699 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:46:00,699 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:00,907 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/bicepv46-media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:00,965 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/4mjpa5w5-events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:01,156 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /tmp/tmpiwoipcdrwandb/57dh8lj3-media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:01,388 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:01,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:01,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:02,389 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:02,607 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:03,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:04,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:05,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:06,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:07,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:07,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:08,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:09,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:10,608 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:11,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:12,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:12,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:13,560 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:46:13,560 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:46:13,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:14,609 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:15,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:16,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:17,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:17,656 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:18,397 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:46:18,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:19,610 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:20,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:21,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:22,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:23,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:23,635 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:24,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:25,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:26,611 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:27,113 DEBUG SenderThread:2243413 [sender.py:send():379] send: exit +2025-01-07 22:46:27,113 INFO SenderThread:2243413 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 22:46:27,114 INFO SenderThread:2243413 [sender.py:send_exit():588] handling runtime: 1019 +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:send_exit():594] send defer +2025-01-07 22:46:27,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,115 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 22:46:27,115 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 22:46:27,115 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 1 +2025-01-07 22:46:27,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,115 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 22:46:27,116 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,116 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 22:46:27,116 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 2 +2025-01-07 22:46:27,116 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 22:46:27,116 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 22:46:27,116 DEBUG SystemMonitor:2243413 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 22:46:27,116 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 22:46:27,117 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 22:46:27,170 INFO HandlerThread:2243413 [interfaces.py:finish():200] Joined network monitor +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 3 +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send():379] send: stats +2025-01-07 22:46:27,171 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,171 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 22:46:27,171 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 22:46:27,171 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 4 +2025-01-07 22:46:27,171 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:27,171 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 22:46:27,402 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:28,402 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:33,699 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 22:46:33,700 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,700 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 22:46:33,700 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 5 +2025-01-07 22:46:33,701 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:33,701 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,701 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 22:46:33,701 DEBUG SenderThread:2243413 [sender.py:send():379] send: history +2025-01-07 22:46:33,702 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:46:33,702 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:33,702 DEBUG SenderThread:2243413 [sender.py:send():379] send: summary +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:46:33,706 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 22:46:33,706 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 6 +2025-01-07 22:46:33,706 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,706 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 22:46:33,706 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 7 +2025-01-07 22:46:33,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:46:33,707 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:33,707 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 22:46:33,707 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:33,707 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 22:46:34,114 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:34,405 INFO Thread-12 :2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:37,029 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 8 +2025-01-07 22:46:37,029 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:37,029 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,030 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 22:46:37,030 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,030 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 22:46:37,030 INFO SenderThread:2243413 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 22:46:37,033 INFO SenderThread:2243413 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 22:46:37,035 INFO SenderThread:2243413 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 22:46:37,048 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 9 +2025-01-07 22:46:37,048 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,048 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 22:46:37,048 DEBUG SenderThread:2243413 [sender.py:send():379] send: artifact +2025-01-07 22:46:37,115 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 22:46:37,285 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 22:46:37,285 INFO SenderThread:2243413 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 22:46:37,406 INFO SenderThread:2243413 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml config.yaml +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json wandb-summary.json +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt requirements.txt +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-metadata.json wandb-metadata.json +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 events.out.tfevents.1736306968.leonmkim-ROG-Strix-G15CS-G15CS.2243246.0 +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log output.log +2025-01-07 22:46:37,407 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/diff.patch diff.patch +2025-01-07 22:46:37,408 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 22:46:37,413 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 media/videos/eval/8_eval_8_3e5641ff60163a911937.mp4 +2025-01-07 22:46:37,413 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 media/videos/eval/14_eval_14_3af06c0a0e058587490f.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 media/videos/eval/11_eval_11_a15ed0619cc72d1cf6a6.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 media/videos/eval/15_eval_15_de33546b235fc7477b2e.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 media/videos/eval/18_eval_18_4987dfaab4d6ad84d0ce.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 media/videos/eval/12_eval_12_64d95a49b894bdfcd37b.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 media/videos/eval/9_eval_9_ef5c54b11ebe40dec8b2.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 media/videos/eval/4_eval_4_da4af638a8cd5e4e155c.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 media/videos/eval/5_eval_5_aa2adef9cffa563b3117.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 media/videos/eval/19_eval_19_3cc271ae74232dc330ac.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 media/videos/eval/6_eval_6_1bdc6ce07f76b86e3067.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 media/videos/eval/1_eval_1_479971f5e97e50638f22.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 media/videos/eval/16_eval_16_991c282f4f03c1a3f74b.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 media/videos/eval/0_eval_0_c0416889a3e77e5f6820.mp4 +2025-01-07 22:46:37,414 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 media/videos/eval/10_eval_10_119d64d622b2c3c1d8e6.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 media/videos/eval/17_eval_17_abf814c17374e1cd6bef.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 media/videos/eval/13_eval_13_c0a7294f80db38ff7cb0.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 media/videos/eval/3_eval_3_4107d85ee8f655917ae7.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 media/videos/eval/7_eval_7_b94820970d3238dd1d30.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 media/videos/eval/2_eval_2_e75d5fcf827136b5c43b.mp4 +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json media/table/eval/total_success_rate_ci_21_fd9e313f895b1255cc98.table.json +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json media/table/eval/success_rate_ci_20_d75e5a883a216df50b02.table.json +2025-01-07 22:46:37,415 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 10 +2025-01-07 22:46:37,415 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:37,415 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,415 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 22:46:37,416 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,416 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 22:46:37,416 INFO SenderThread:2243413 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:46:37,601 INFO wandb-upload_0:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/wandb-summary.json +2025-01-07 22:46:37,644 INFO wandb-upload_2:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/requirements.txt +2025-01-07 22:46:37,659 INFO wandb-upload_1:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/config.yaml +2025-01-07 22:46:37,688 INFO wandb-upload_3:2243413 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/files/output.log +2025-01-07 22:46:37,888 INFO Thread-11 (_thread_body):2243413 [sender.py:transition_state():614] send defer: 11 +2025-01-07 22:46:37,889 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,889 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 22:46:37,889 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,889 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 22:46:37,889 INFO SenderThread:2243413 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 12 +2025-01-07 22:46:37,892 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:37,892 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 22:46:37,892 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 22:46:37,892 INFO SenderThread:2243413 [file_stream.py:finish():601] file stream finish called +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [file_stream.py:finish():605] file stream finish is done +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 13 +2025-01-07 22:46:38,083 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:38,083 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 22:46:38,083 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 22:46:38,083 INFO SenderThread:2243413 [sender.py:transition_state():614] send defer: 14 +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send():379] send: final +2025-01-07 22:46:38,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: defer +2025-01-07 22:46:38,084 INFO HandlerThread:2243413 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send():379] send: footer +2025-01-07 22:46:38,084 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: defer +2025-01-07 22:46:38,084 INFO SenderThread:2243413 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 22:46:38,084 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:38,085 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 22:46:38,085 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:38,085 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: server_info +2025-01-07 22:46:38,086 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 22:46:38,087 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 22:46:38,087 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 22:46:38,088 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:46:38,139 DEBUG SenderThread:2243413 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 22:46:38,139 INFO MainThread:2243413 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 22:46:38,140 INFO MainThread:2243413 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 22:46:38,140 INFO MainThread:2243413 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 22:46:38,140 DEBUG HandlerThread:2243413 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 22:46:38,140 INFO HandlerThread:2243413 [handler.py:finish():882] shutting down handler +2025-01-07 22:46:39,087 INFO WriterThread:2243413 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [sender.py:finish():1615] shutting down sender +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 22:46:39,139 INFO SenderThread:2243413 [file_pusher.py:join():175] waiting for file pusher diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug.log b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f6047abe6c6f1f19eaea1c25738b18c6a1bd5031 --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Configure stats pid to 2243246 +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/settings +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 22:29:27,688 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug.log +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/logs/debug-internal.log +2025-01-07 22:29:27,689 INFO MainThread:2243246 [wandb_init.py:init():569] calling init triggers +2025-01-07 22:29:27,690 INFO MainThread:2243246 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg'} +2025-01-07 22:29:27,691 INFO MainThread:2243246 [wandb_init.py:init():619] starting backend +2025-01-07 22:29:27,691 INFO MainThread:2243246 [wandb_init.py:init():623] setting up manager +2025-01-07 22:29:27,695 INFO MainThread:2243246 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 22:29:27,696 INFO MainThread:2243246 [wandb_init.py:init():631] backend started and connected +2025-01-07 22:29:27,707 INFO MainThread:2243246 [wandb_init.py:init():720] updated telemetry +2025-01-07 22:29:27,713 INFO MainThread:2243246 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 22:29:28,003 INFO MainThread:2243246 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 22:29:28,077 INFO MainThread:2243246 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 22:29:28,077 INFO MainThread:2243246 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 22:29:28,408 INFO MainThread:2243246 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 22:29:28,408 INFO MainThread:2243246 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 22:29:28,409 INFO MainThread:2243246 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 22:29:28,409 INFO MainThread:2243246 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 22:29:28,410 INFO MainThread:2243246 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 22:29:28,410 INFO MainThread:2243246 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/opf0nkwg/tb, True +2025-01-07 22:29:32,802 INFO MainThread:2243246 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 22:46:39,141 WARNING MsgRouterThr:2243246 [router.py:message_loop():77] message_loop has been closed diff --git a/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b76cc111433c3c70c389e152e934b73a144fa6be --- /dev/null +++ b/opf0nkwg/wandb/run-20250107_222927-opf0nkwg/run-opf0nkwg.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed99bb57465f25a8e67e1e60ddd0fbd8d2bc81d45ac38289a5546347ac626d2 +size 241420 diff --git a/rgxendfg/.hydra/config.yaml b/rgxendfg/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f42ede41d78bb1da81f2d953488ed95ba1988804 --- /dev/null +++ b/rgxendfg/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/rgxendfg/.hydra/hydra.yaml b/rgxendfg/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..323fc5ec353b08f166de186fba2faefdf09d22c7 --- /dev/null +++ b/rgxendfg/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/rgxendfg/.hydra/overrides.yaml b/rgxendfg/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/rgxendfg/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/rgxendfg/episode_rosbags/aligned_depth_to_color_K.npy b/rgxendfg/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/rgxendfg/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/rgxendfg/episode_rosbags/cam_tf_world.npy b/rgxendfg/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/rgxendfg/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/rgxendfg/episode_rosbags/color_K.npy b/rgxendfg/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/rgxendfg/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/rgxendfg/episode_rosbags/depth_K.npy b/rgxendfg/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/rgxendfg/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/rgxendfg/eval_robot.log b/rgxendfg/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..d76492790b7a892573163e4855de93f0c41f64b8 --- /dev/null +++ b/rgxendfg/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:54:41,831][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:54:41,834][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/rgxendfg/eval_video/0_eval.mp4 b/rgxendfg/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8fd0451a7b708c217d9c40678ddd757265785dbf --- /dev/null +++ b/rgxendfg/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99588144472d6a23cee839ba5adaf32bbaaa9905d72dbdb4c2952a66645b0d1 +size 835180 diff --git a/rgxendfg/eval_video/1_eval.mp4 b/rgxendfg/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45ff8da8e8060c6ea5adee4d6bfd50efac981f0d --- /dev/null +++ b/rgxendfg/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ac1d3949294c0444e4261673076656e5a2b66a2e17800b953fb39e9ab5aca8 +size 1138649 diff --git a/rgxendfg/eval_video/2_eval.mp4 b/rgxendfg/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..104354282cef77e2d8ba0476f6c7f2de06d6417b --- /dev/null +++ b/rgxendfg/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:801e83780d634b6f5c3a674e26cc98e6f61937b5d46431991b46499490e718ed +size 1293814 diff --git a/rgxendfg/eval_video/3_eval.mp4 b/rgxendfg/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be7fdcb97a276e195f46e70e597dbf12de85346f --- /dev/null +++ b/rgxendfg/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5db1b98db4e00caa96ceabd44f4b9a26c219ffda57dc6595474d5e5530b5cfce +size 1260806 diff --git a/rgxendfg/eval_video/4_eval.mp4 b/rgxendfg/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c79985763af5feab1dcc56da3438f75ed1db0bde --- /dev/null +++ b/rgxendfg/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54e419694f010e143b0fc16eb54cf68607e0cc120ba0cd3348f3e25882a0c30 +size 909065 diff --git a/rgxendfg/tb/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 b/rgxendfg/tb/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 new file mode 100644 index 0000000000000000000000000000000000000000..e2058cdca8d09079c58112a273fb3c4dbc6862fa --- /dev/null +++ b/rgxendfg/tb/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec0de8702f030ee9491a9194ec222461bab3b3055b54e34b06eb187335dd4efc +size 6519 diff --git a/rgxendfg/wandb/debug-internal.log b/rgxendfg/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2c6481e642c2977487c5e8d87f18311602b23b87 --- /dev/null +++ b/rgxendfg/wandb/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-07 20:54:47,564 INFO StreamThr :2233136 [internal.py:wandb_internal():85] W&B internal server running at pid: 2233136, started at: 2025-01-07 20:54:47.564098 +2025-01-07 20:54:47,565 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:54:47,578 INFO WriterThread:2233136 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb +2025-01-07 20:54:47,578 DEBUG SenderThread:2233136 [sender.py:send():379] send: header +2025-01-07 20:54:47,587 DEBUG SenderThread:2233136 [sender.py:send():379] send: run +2025-01-07 20:54:47,801 INFO SenderThread:2233136 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files +2025-01-07 20:54:47,801 INFO SenderThread:2233136 [sender.py:_start_run_threads():1188] run started: rgxendfg with start time 1736301287.565172 +2025-01-07 20:54:47,816 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:54:47,816 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:54:47,891 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:54:47,896 DEBUG HandlerThread:2233136 [system_info.py:__init__():26] System info init +2025-01-07 20:54:47,896 DEBUG HandlerThread:2233136 [system_info.py:__init__():41] System info init done +2025-01-07 20:54:47,896 INFO HandlerThread:2233136 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:54:47,896 INFO SystemMonitor:2233136 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:54:47,896 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:54:47,897 INFO HandlerThread:2233136 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:54:47,897 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:54:47,898 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:54:47,899 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:54:47,899 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:54:47,910 DEBUG HandlerThread:2233136 [system_info.py:probe():152] Probing system +2025-01-07 20:54:47,912 DEBUG HandlerThread:2233136 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:probe():200] Probing system done +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:54:47.910550', 'startedAt': '2025-01-08T01:54:47.548886', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4932499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 620.0248336791992}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:54:47,916 INFO HandlerThread:2233136 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:54:47,916 INFO HandlerThread:2233136 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:_save_code():44] Saving code +2025-01-07 20:54:47,918 DEBUG HandlerThread:2233136 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:54:47,918 DEBUG HandlerThread:2233136 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:54:47,981 DEBUG HandlerThread:2233136 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:54:47,981 INFO HandlerThread:2233136 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:54:47,985 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:54:48,238 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:54:48,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:54:48,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:54:48,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:54:48,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:48,241 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/9tbi3wx7-diff.patch +2025-01-07 20:54:48,241 INFO HandlerThread:2233136 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/tb" + save: true +} +control { + relay_id: "4832a03cec4a4398bb9d99a602c3d362" +} +_info { + stream_id: "rgxendfg" +} + +2025-01-07 20:54:48,242 DEBUG HandlerThread:2233136 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:54:48,248 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/7orbho4p-wandb-metadata.json +2025-01-07 20:54:48,260 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/72cff047-code/FISH/eval_robot.py +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: telemetry +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: tbrecord +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:54:48,299 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 with policy live +2025-01-07 20:54:48,474 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/epzd6j7h-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH +2025-01-07 20:54:49,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:50,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:50,802 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:51,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:52,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:52,460 DEBUG SenderThread:2233136 [sender.py:send():379] send: config +2025-01-07 20:54:53,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:53,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:54,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:54,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:55,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:56,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:57,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:58,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:58,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:58,804 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:59,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:00,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:01,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:02,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:02,805 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:03,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:03,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:03,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:03,308 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:04,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:04,806 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:05,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:06,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:06,807 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:07,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:08,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:08,807 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:09,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:09,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:10,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:11,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:12,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:13,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:14,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:14,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:15,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:16,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:17,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:18,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:18,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:18,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:19,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:19,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:19,810 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:55:20,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:21,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:22,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:23,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:24,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:25,108 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:25,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:25,369 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:55:25,370 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 with policy now +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:55:25,371 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:55:25,585 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/unbth1wl-media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media +2025-01-07 20:55:26,063 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ltldjmmf-media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:26,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:26,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:27,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:28,244 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:29,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:30,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:30,814 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:31,097 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:31,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:32,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:32,815 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:33,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:33,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:33,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:34,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:35,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:36,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:36,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:37,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:38,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:39,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:40,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:41,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:41,483 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:41,818 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:55:42,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:42,818 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:43,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:44,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:44,819 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:45,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:46,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:46,820 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:47,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:47,443 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:47,899 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:55:47,900 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:55:48,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:48,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:48,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:49,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:50,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:51,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:52,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:53,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:53,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:54,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:55,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:56,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:57,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:58,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:58,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:59,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:00,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:01,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:02,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:03,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:03,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:03,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:03,386 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:04,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:04,826 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:05,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:06,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:06,827 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:07,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:08,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:09,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:09,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:10,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:11,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:12,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:13,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:13,561 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:56:13,565 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:56:13,565 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 with policy now +2025-01-07 20:56:13,565 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:56:13,566 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:56:13,567 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:56:13,781 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/qizaab53-media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:56:14,091 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/3l55y6ad-media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:14,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:14,567 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:14,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:15,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:15,830 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:16,259 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:17,260 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:17,900 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:56:18,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:18,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:18,261 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:18,831 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:19,258 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:20,044 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:20,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:21,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:22,266 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:22,832 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:23,258 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:24,267 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:25,255 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:25,265 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:26,266 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:27,268 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:27,716 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:56:27,716 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:56:27,717 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:56:27,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:56:27,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:56:28,082 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/akr89wud-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:56:28,269 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:28,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:29,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:30,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:30,377 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:30,835 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:31,275 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:32,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:32,835 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:33,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:33,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:33,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:34,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:34,836 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:35,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:36,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:36,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:36,837 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:37,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:38,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:39,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:40,275 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:41,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:41,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:42,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:43,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:44,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:44,839 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:45,290 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:46,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:46,945 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:47,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:47,901 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:56:48,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:48,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:48,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:49,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:50,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:51,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:52,253 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:52,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:52,842 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:53,289 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:54,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:55,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:56,283 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:57,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:57,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:58,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:58,844 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:59,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:00,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:01,291 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:01,541 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:57:01,541 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:57:01,541 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 with policy now +2025-01-07 20:57:01,542 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:01,542 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:01,542 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:01,769 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/hrcj4vaj-media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:57:02,118 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ecjsfkzd-media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:02,297 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:02,563 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:02,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:03,244 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:03,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:03,303 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:04,296 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:05,313 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:06,295 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:06,846 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:07,299 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:08,111 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:08,294 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:09,299 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:10,301 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:10,848 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:11,308 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:12,304 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:13,263 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:13,315 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:14,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:15,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:16,311 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:17,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:17,500 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:17,500 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:17,500 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:17,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:17,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:57:17,901 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:57:18,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:18,246 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:18,314 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:18,386 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:18,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:19,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:20,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:20,851 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:21,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:22,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:22,852 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:23,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:24,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:24,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:24,853 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:25,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:26,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:27,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:28,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:29,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:30,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:30,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:31,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:32,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:33,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:33,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:33,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:34,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:35,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:35,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:36,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:37,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:38,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:39,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:40,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:41,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:41,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:42,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:43,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:44,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:45,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:46,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:47,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:47,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:47,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:57:48,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:48,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:48,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:49,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:50,334 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:50,861 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:50,949 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:57:50,949 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 with policy now +2025-01-07 20:57:50,952 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:57:50,953 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:50,953 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:50,953 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:51,171 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/jfbqzpo7-media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:51,328 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:57:52,148 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ruz5h453-media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:52,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:52,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:53,005 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:53,334 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:54,335 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:55,333 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:56,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:56,863 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:57,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:58,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:58,333 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:58,864 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:59,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:00,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:01,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:02,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:03,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:03,249 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:03,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:03,370 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:04,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:05,341 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:06,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:07,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:08,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:09,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:09,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:10,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:10,566 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:10,566 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:10,567 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:10,868 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:10,869 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:11,132 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/gzq3p68q-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:11,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:12,347 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:12,869 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:13,341 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:14,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:14,347 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:15,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:16,342 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:16,871 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:17,346 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:17,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:18,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:18,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:18,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:18,871 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:19,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:20,313 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:20,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:21,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:22,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:23,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:24,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:25,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:26,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:26,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:27,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:28,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:29,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:30,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:31,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:31,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:32,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:33,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:33,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:33,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:34,346 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:35,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:35,647 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:35,647 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 with policy now +2025-01-07 20:58:35,657 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:35,657 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:35,657 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:35,658 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:35,877 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:35,877 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:35,878 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:58:35,900 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/5epie9ok-media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:36,165 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/4j3vtuu9-media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:36,349 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:36,683 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:36,878 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:37,351 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:38,356 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:39,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:40,350 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:40,879 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:41,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:42,136 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:42,458 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:43,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:44,451 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:44,880 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:45,451 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:46,449 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:47,292 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:47,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:47,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:48,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:48,250 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:48,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:49,457 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:50,466 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:51,306 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:58:51,306 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:58:51,456 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:51,567 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:51,827 INFO wandb-upload_1:2233136 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9iely7im +2025-01-07 20:58:51,883 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table +2025-01-07 20:58:51,885 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval +2025-01-07 20:58:52,087 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:58:52,096 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:52,198 INFO SenderThread:2233136 [sender.py:send_request_log_artifact():1525] logged artifact run-rgxendfg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQzMjU3MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0NDc0Mw==', 'latestArtifact': None}} +2025-01-07 20:58:52,198 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:52,198 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 20:58:52,199 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:52,199 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,213 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,214 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send():379] send: telemetry +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,230 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,230 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,244 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,244 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,245 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,255 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,255 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,266 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,267 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,279 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,279 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,280 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,280 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,290 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,290 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,309 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,309 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,309 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:52,325 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,325 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,326 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,326 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:58:52,391 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/gehzqjyy-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:52,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:52,653 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/kkz5ua2t-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval +2025-01-07 20:58:53,038 INFO wandb-upload_2:2233136 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdy65qdge +2025-01-07 20:58:53,172 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/pcbp8hgv-media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:53,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:53,543 INFO SenderThread:2233136 [sender.py:send_request_log_artifact():1525] logged artifact run-rgxendfg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQzMjU5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0NDc0Ng==', 'latestArtifact': None}} +2025-01-07 20:58:53,543 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:53,544 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json with policy now +2025-01-07 20:58:53,544 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:53,544 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:53,544 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:53,805 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ahy4g0j3-media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:53,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:54,456 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:54,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:55,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:56,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:57,427 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:57,473 DEBUG SenderThread:2233136 [sender.py:send():379] send: exit +2025-01-07 20:58:57,473 INFO SenderThread:2233136 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_exit():588] handling runtime: 249 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_exit():594] send defer +2025-01-07 20:58:57,474 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,474 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:58:57,474 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:58:57,474 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,474 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:58:57,475 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,475 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:58:57,475 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:58:57,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:58:57,475 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:58:57,475 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:58:57,476 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:58:57,528 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:58:57,528 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:58:57,529 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:58:57,529 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,529 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:58:57,529 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:58:57,529 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:57,529 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,529 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:58:57,530 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,530 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:58:57,530 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:58:57,530 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,530 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:58:57,886 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:58,886 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,627 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:59:03,627 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:03,627 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:59:03,627 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:59:03,629 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:59:03,629 DEBUG SenderThread:2233136 [sender.py:send():379] send: summary +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:59:03,630 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:59:03,630 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,630 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:59:03,630 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:59:03,634 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:59:03,770 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:59:03,770 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,770 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:59:03,770 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,770 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:59:03,888 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:59:03,888 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:59:04,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:05,727 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:59:05,727 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:05,728 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:05,728 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:59:05,728 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:05,728 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:59:05,728 INFO SenderThread:2233136 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:59:05,730 INFO SenderThread:2233136 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:59:05,732 INFO SenderThread:2233136 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:59:05,747 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:59:05,747 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:05,747 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:59:05,748 DEBUG SenderThread:2233136 [sender.py:send():379] send: artifact +2025-01-07 20:59:05,889 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:59:06,021 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:59:06,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml config.yaml +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json wandb-summary.json +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt requirements.txt +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log output.log +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch diff.patch +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:59:06,895 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:59:06,899 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:06,900 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:06,900 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:59:06,900 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:06,900 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:59:06,900 INFO SenderThread:2233136 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:59:07,060 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:59:07,084 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:59:07,120 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt +2025-01-07 20:59:07,125 INFO wandb-upload_4:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:07,154 INFO wandb-upload_3:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/oj037jl0-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:59:07,354 INFO Thread-11 (_thread_body):2233136 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:59:07,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,354 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:59:07,355 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,355 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:59:07,355 INFO SenderThread:2233136 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:59:07,356 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,356 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:59:07,356 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:59:07,476 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,536 INFO SenderThread:2233136 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:59:07,536 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:59:07,536 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,536 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,536 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send():379] send: final +2025-01-07 20:59:07,537 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,537 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send():379] send: footer +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:59:07,538 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,538 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,538 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:59:07,539 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:59:07,541 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:59:07,594 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:59:07,595 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:59:07,595 INFO HandlerThread:2233136 [handler.py:finish():882] shutting down handler +2025-01-07 20:59:08,539 INFO WriterThread:2233136 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [sender.py:finish():1615] shutting down sender +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [file_pusher.py:join():175] waiting for file pusher diff --git a/rgxendfg/wandb/debug.log b/rgxendfg/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e2872a7314d1662aeee5f17a3cd832b790c265e0 --- /dev/null +++ b/rgxendfg/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Configure stats pid to 2232948 +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/settings +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug.log +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug-internal.log +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg'} +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():619] starting backend +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():623] setting up manager +2025-01-07 20:54:47,563 INFO MainThread:2232948 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:54:47,564 INFO MainThread:2232948 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:54:47,577 INFO MainThread:2232948 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:54:47,585 INFO MainThread:2232948 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:54:47,816 INFO MainThread:2232948 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:54:47,886 INFO MainThread:2232948 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:54:47,886 INFO MainThread:2232948 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:54:48,241 INFO MainThread:2232948 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:54:48,241 INFO MainThread:2232948 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/tb, True +2025-01-07 20:54:52,459 INFO MainThread:2232948 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:59:08,595 WARNING MsgRouterThr:2232948 [router.py:message_loop():77] message_loop has been closed diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3cf10e508e06dd849c8574243edb2f20053df0b --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736301287 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1af31b2ff07685aba62f0764c3073e6e9867fe4d --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch @@ -0,0 +1,21 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..6e694c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..53134a21059a2f437e9e6358977c4edfee3d6f16 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[31, 60, 0.5166666666666667, 0.36618638041311524, 0.6638777851730655, 0.39421035351484845, 0.6469452678863367, 0.4134980850094706, 0.6186398193002887, 0.42932465301377465, 0.6043455861300154]]} \ No newline at end of file diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8fd0451a7b708c217d9c40678ddd757265785dbf --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99588144472d6a23cee839ba5adaf32bbaaa9905d72dbdb4c2952a66645b0d1 +size 835180 diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45ff8da8e8060c6ea5adee4d6bfd50efac981f0d --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ac1d3949294c0444e4261673076656e5a2b66a2e17800b953fb39e9ab5aca8 +size 1138649 diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..104354282cef77e2d8ba0476f6c7f2de06d6417b --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:801e83780d634b6f5c3a674e26cc98e6f61937b5d46431991b46499490e718ed +size 1293814 diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..be7fdcb97a276e195f46e70e597dbf12de85346f --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5db1b98db4e00caa96ceabd44f4b9a26c219ffda57dc6595474d5e5530b5cfce +size 1260806 diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c79985763af5feab1dcc56da3438f75ed1db0bde --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54e419694f010e143b0fc16eb54cf68607e0cc120ba0cd3348f3e25882a0c30 +size 909065 diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..eb95c39a668cbdd0ef1ddb61bc5b5d1405dbf530 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736301292.461635]: resetting environment +[INFO] [1736301292.465124]: cleared current plan +[INFO] [1736301292.466417]: moving to home +[INFO] [1736301293.469064]: reached home +[INFO] [1736301293.470593]: reset action history +[INFO] [1736301296.183050]: environment reset +Starting episode 0 +[INFO] [1736301296.185064]: resetting environment +[INFO] [1736301296.188394]: cleared current plan +[INFO] [1736301296.189333]: moving to home +[INFO] [1736301297.191665]: reached home +[INFO] [1736301297.193262]: reset action history +[INFO] [1736301299.950943]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736301301.868499]: resetting environment +[INFO] [1736301301.875048]: cleared current plan +[INFO] [1736301301.876672]: moving to home +[INFO] [1736301302.879893]: reached home +[INFO] [1736301302.881067]: reset action history +[INFO] [1736301305.586536]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/episode_rosbags/episode_0_2025-01-07-20-55-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2232948.log +started roslaunch server http://158.130.50.37:40689/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2233260] +process[print_text-2]: started with pid [2233262] +process[rosbag_record-3]: started with pid [2233263] +started bagging! +preempting episode +[INFO] [1736301324.342153]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1736301325.107791]: Stored episode 1. +[INFO] [1736301325.107997]: Saving video... +[INFO] [1736301325.328166]: Video saved! +Starting episode 1 +[INFO] [1736301325.339811]: resetting environment +[INFO] [1736301325.350765]: cleared current plan +[INFO] [1736301325.351632]: moving to home +[INFO] [1736301329.055701]: reached home +[INFO] [1736301329.056350]: reset action history +[INFO] [1736301331.766676]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301341.487228]: resetting environment +[INFO] [1736301341.490277]: cleared current plan +[INFO] [1736301341.490481]: moving to home +[INFO] [1736301342.492071]: reached home +[INFO] [1736301342.492690]: reset action history +[INFO] [1736301345.197954]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/episode_rosbags/episode_1_2025-01-07-20-55-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2232948.log +started roslaunch server http://158.130.50.37:38673/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2233373] +process[print_text-5]: started with pid [2233374] +process[rosbag_record-6]: started with pid [2233398] +started bagging! +For topic gripper_width: timestamp difference is 100285706 for nearest: 1736301362769318041 - target: 1736301362669032335 at idx 249 +For topic gripper_width: timestamp difference is 101041619 for nearest: 1736301363769317452 - target: 1736301363668275833 at idx 249 +For topic gripper_width: timestamp difference is 102369092 for nearest: 1736301365369318984 - target: 1736301365266949892 at idx 249 +[INFO] [1736301372.152011]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736301372.937987]: Stored episode 2. +[INFO] [1736301372.938657]: Saving video... +[INFO] [1736301373.515956]: Video saved! +Starting episode 2 +[INFO] [1736301373.547401]: resetting environment +[INFO] [1736301373.557360]: cleared current plan +[INFO] [1736301373.565196]: moving to home +[WARN] [1736301373.537645]: Plan exhausted +[INFO] [1736301377.271560]: reached home +[INFO] [1736301377.274949]: reset action history +[INFO] [1736301380.043828]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301387.313683]: resetting environment +[INFO] [1736301387.330951]: cleared current plan +[INFO] [1736301387.331255]: moving to home +[INFO] [1736301388.333813]: reached home +[INFO] [1736301388.334444]: reset action history +[INFO] [1736301391.063175]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/episode_rosbags/episode_2_2025-01-07-20-56-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2232948.log +started roslaunch server http://158.130.50.37:35823/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2233533] +process[print_text-8]: started with pid [2233534] +process[rosbag_record-9]: started with pid [2233535] +started bagging! +For topic gripper_width: timestamp difference is -106624904 for nearest: 1736301395002650199 - target: 1736301395109275103 at idx 54 +For topic gripper_width: timestamp difference is 100474101 for nearest: 1736301403902653676 - target: 1736301403802179575 at idx 161 +For topic gripper_width: timestamp difference is 106244661 for nearest: 1736301411202649213 - target: 1736301411096404552 at idx 249 +For topic gripper_width: timestamp difference is 111024554 for nearest: 1736301417269318516 - target: 1736301417158293962 at idx 249 +[INFO] [1736301420.376652]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736301421.227649]: Stored episode 3. +[INFO] [1736301421.235464]: Saving video... +[INFO] [1736301421.497097]: Video saved! +Starting episode 3 +[INFO] [1736301421.545181]: resetting environment +[INFO] [1736301421.559035]: cleared current plan +[INFO] [1736301421.562601]: moving to home +[INFO] [1736301425.271703]: reached home +[INFO] [1736301425.279178]: reset action history +[INFO] [1736301428.109817]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301437.374076]: resetting environment +[INFO] [1736301437.407617]: cleared current plan +[INFO] [1736301437.417297]: moving to home +[INFO] [1736301438.426175]: reached home +[INFO] [1736301438.428017]: reset action history +[INFO] [1736301441.210646]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/episode_rosbags/episode_3_2025-01-07-20-57-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2232948.log +started roslaunch server http://158.130.50.37:42729/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2233672] +process[print_text-11]: started with pid [2233673] +process[rosbag_record-12]: started with pid [2233674] +started bagging! +[INFO] [1736301469.839356]: Storing episode... +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[INFO] [1736301470.648924]: Stored episode 4. +[INFO] [1736301470.657076]: Saving video... +[INFO] [1736301470.909889]: Video saved! +Starting episode 4 +[INFO] [1736301470.963120]: resetting environment +[INFO] [1736301470.987865]: cleared current plan +[INFO] [1736301471.002530]: moving to home +[INFO] [1736301474.811035]: reached home +[INFO] [1736301474.821195]: reset action history +[INFO] [1736301477.650868]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736301490.490738]: resetting environment +[INFO] [1736301490.516181]: cleared current plan +[INFO] [1736301490.520223]: moving to home +[INFO] [1736301491.528081]: reached home +[INFO] [1736301491.539089]: reset action history +[INFO] [1736301494.344597]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/episode_rosbags/episode_4_2025-01-07-20-58-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2232948.log +started roslaunch server http://158.130.50.37:37135/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2233811] +process[print_text-14]: started with pid [2233812] +process[rosbag_record-15]: started with pid [2233813] +started bagging! +preempting episode +[INFO] [1736301514.588401]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736301515.363287]: Stored episode 5. +[INFO] [1736301515.367549]: Saving video... +[INFO] [1736301515.593204]: Video saved! +[INFO] [1736301515.661156]: resetting environment +[INFO] [1736301515.676348]: cleared current plan +[INFO] [1736301515.681866]: moving to home +[INFO] [1736301519.289829]: reached home +[INFO] [1736301519.294673]: reset action history +[INFO] [1736301522.135269]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3b35fb98da4d5b5999e5694de90798523bb38643 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:54:47.910550", + "startedAt": "2025-01-08T01:54:47.548886", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.4932499999995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.976, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.02, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 620.0248336791992 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4bc8e7edc2fafcb5c4c1c6a7e8ff5a98767d99af --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c99588144472d6a23cee839ba5adaf32bbaaa9905d72dbdb4c2952a66645b0d1", "size": 835180, "path": "media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4"}, "global_step": 575, "_timestamp": 1736301531.9252892, "_runtime": 244.36011719703674, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a3ac1d3949294c0444e4261673076656e5a2b66a2e17800b953fb39e9ab5aca8", "size": 1138649, "path": "media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4"}, "eval/num_success": 1.0, "episode": 60.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "801e83780d634b6f5c3a674e26cc98e6f61937b5d46431991b46499490e718ed", "size": 1293814, "path": "media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5db1b98db4e00caa96ceabd44f4b9a26c219ffda57dc6595474d5e5530b5cfce", "size": 1260806, "path": "media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b54e419694f010e143b0fc16eb54cf68607e0cc120ba0cd3348f3e25882a0c30", "size": 909065, "path": "media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://c0n8nxhxeb473ynfrco3wtks4mea4w5fi5zphg0scamy7hjrnrgyvxr9y7gsc9e9kedrialg39hyvknll71l5u8mse0p6f2wp4cdffidumz01eoxgbuljvbxdkp0xqcf/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://rly2zak7dfmkhemguksyk9jkpuux8hvdgejdy4rfk2c6d3v0bp2le6705bkz3mcvbblfe8uthw5soi6eti57b2hznllso0un4hstg3mj0sy53gz8glk3lvlhr57yu0yg:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 31, "total_num_episodes": 60, "total_success_rate": 0.5166666666666667, "total_success_rate_lb_0.01": 0.36618638041311524, "total_success_rate_ub_0.01": 0.6638777851730655, "total_success_rate_lb_0.025": 0.39421035351484845, "total_success_rate_ub_0.025": 0.6469452678863367, "total_success_rate_lb_0.05": 0.4134980850094706, "total_success_rate_ub_0.05": 0.6186398193002887, "total_success_rate_lb_0.1": 0.42932465301377465, "total_success_rate_ub_0.1": 0.6043455861300154, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "26598ad0ab173325629d941b5290cec1177a3eba1844de237e492da73e958c69", "size": 523, "artifact_path": "wandb-client-artifact://k220ux9h4qamukuaiz4jvhl1ya2sq4fdezqer1dlct1zhwa68su8c2773grj1w4r58ry2q5kdvlr3nkatnwfntfmrauxcfphq74kl7mmrjl61lg38ri8ezb5csbax7s3/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://aqnlu2hm761fdd1umegyqmgf4oumwwdusm4snd42i6l9l7aqmf3mmqqa2pfpn54cr29q1maav7d35e13vtmgtal7hplv6re2yqz01usl7iflv82fka3az87h17vc6kp4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 249}, "eval/total_success_rate": 0.5166666507720947, "eval/total_success_rate_lb_0.01": 0.36618638038635254, "eval/total_success_rate_ub_0.01": 0.6638777852058411, "eval/total_success_rate_lb_0.025": 0.3942103683948517, "eval/total_success_rate_ub_0.025": 0.6469452381134033, "eval/total_success_rate_lb_0.05": 0.41349807381629944, "eval/total_success_rate_ub_0.05": 0.6186398267745972, "eval/total_success_rate_lb_0.1": 0.4293246567249298, "eval/total_success_rate_ub_0.1": 0.6043455600738525} \ No newline at end of file diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug-internal.log b/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2c6481e642c2977487c5e8d87f18311602b23b87 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-07 20:54:47,564 INFO StreamThr :2233136 [internal.py:wandb_internal():85] W&B internal server running at pid: 2233136, started at: 2025-01-07 20:54:47.564098 +2025-01-07 20:54:47,565 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:54:47,578 INFO WriterThread:2233136 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb +2025-01-07 20:54:47,578 DEBUG SenderThread:2233136 [sender.py:send():379] send: header +2025-01-07 20:54:47,587 DEBUG SenderThread:2233136 [sender.py:send():379] send: run +2025-01-07 20:54:47,801 INFO SenderThread:2233136 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files +2025-01-07 20:54:47,801 INFO SenderThread:2233136 [sender.py:_start_run_threads():1188] run started: rgxendfg with start time 1736301287.565172 +2025-01-07 20:54:47,816 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:54:47,816 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:54:47,891 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:54:47,896 DEBUG HandlerThread:2233136 [system_info.py:__init__():26] System info init +2025-01-07 20:54:47,896 DEBUG HandlerThread:2233136 [system_info.py:__init__():41] System info init done +2025-01-07 20:54:47,896 INFO HandlerThread:2233136 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:54:47,896 INFO SystemMonitor:2233136 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:54:47,896 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:54:47,897 INFO HandlerThread:2233136 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:54:47,897 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:54:47,898 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:54:47,899 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:54:47,899 INFO SystemMonitor:2233136 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:54:47,910 DEBUG HandlerThread:2233136 [system_info.py:probe():152] Probing system +2025-01-07 20:54:47,912 DEBUG HandlerThread:2233136 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:probe():200] Probing system done +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:54:47.910550', 'startedAt': '2025-01-08T01:54:47.548886', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4932499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.976, 'min': 800.0, 'max': 4700.0}, {'current': 4500.02, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 620.0248336791992}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:54:47,916 INFO HandlerThread:2233136 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:54:47,916 INFO HandlerThread:2233136 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:54:47,916 DEBUG HandlerThread:2233136 [system_info.py:_save_code():44] Saving code +2025-01-07 20:54:47,918 DEBUG HandlerThread:2233136 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:54:47,918 DEBUG HandlerThread:2233136 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:54:47,981 DEBUG HandlerThread:2233136 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:54:47,981 INFO HandlerThread:2233136 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:54:47,985 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:54:47,985 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:54:48,238 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:54:48,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:54:48,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:54:48,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:54:48,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:48,241 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/9tbi3wx7-diff.patch +2025-01-07 20:54:48,241 INFO HandlerThread:2233136 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/tb" + save: true +} +control { + relay_id: "4832a03cec4a4398bb9d99a602c3d362" +} +_info { + stream_id: "rgxendfg" +} + +2025-01-07 20:54:48,242 DEBUG HandlerThread:2233136 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:54:48,248 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/7orbho4p-wandb-metadata.json +2025-01-07 20:54:48,260 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/72cff047-code/FISH/eval_robot.py +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: telemetry +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: tbrecord +2025-01-07 20:54:48,299 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:54:48,299 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 with policy live +2025-01-07 20:54:48,474 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/epzd6j7h-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code +2025-01-07 20:54:48,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH +2025-01-07 20:54:49,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:50,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:50,802 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:51,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:52,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:52,460 DEBUG SenderThread:2233136 [sender.py:send():379] send: config +2025-01-07 20:54:53,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:53,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:54,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:54,803 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:55,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:56,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:57,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:58,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:54:58,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:54:58,804 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:54:59,240 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:00,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:01,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:02,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:02,805 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:03,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:03,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:03,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:03,308 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:04,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:04,806 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:05,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:06,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:06,807 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:07,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:08,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:08,807 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:09,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:09,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:10,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:11,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:12,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:13,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:14,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:14,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:15,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:16,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:17,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:18,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:18,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:18,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:19,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:19,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:19,810 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:55:20,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:21,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:22,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:23,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:24,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:25,108 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:25,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:25,369 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:55:25,370 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 with policy now +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:55:25,370 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:55:25,371 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:55:25,585 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/unbth1wl-media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos +2025-01-07 20:55:25,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media +2025-01-07 20:55:26,063 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ltldjmmf-media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:55:26,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:26,813 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:27,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:28,244 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:29,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:30,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:30,814 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:31,097 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:31,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:32,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:32,815 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:33,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:33,239 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:33,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:34,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:35,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:36,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:36,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:37,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:38,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:39,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:40,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:41,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:41,483 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:41,818 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:55:42,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:42,818 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:43,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:44,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:44,819 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:45,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:46,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:46,820 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:55:47,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:47,443 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:47,899 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:55:47,900 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:55:48,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:55:48,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:55:48,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:49,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:50,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:51,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:52,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:53,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:53,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:54,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:55,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:56,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:57,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:58,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:55:58,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:55:59,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:00,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:01,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:02,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:03,239 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:03,240 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:03,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:03,386 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:04,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:04,826 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:05,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:06,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:06,827 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:07,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:08,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:09,247 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:09,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:10,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:11,251 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:12,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:13,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:13,561 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:56:13,565 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:56:13,565 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 with policy now +2025-01-07 20:56:13,565 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:56:13,566 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:56:13,567 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:56:13,781 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/qizaab53-media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:13,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:56:14,091 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/3l55y6ad-media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:56:14,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:14,567 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:14,829 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:15,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:15,830 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:16,259 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:17,260 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:17,900 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:56:18,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:18,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:18,261 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:18,831 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:19,258 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:20,044 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:20,256 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:21,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:22,266 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:22,832 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:23,258 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:24,267 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:25,255 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:25,265 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:26,266 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:27,268 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:27,716 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:56:27,716 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:56:27,717 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:56:27,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:56:27,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:56:28,082 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/akr89wud-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:56:28,269 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:28,834 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:29,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:30,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:30,377 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:30,835 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:31,275 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:32,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:32,835 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:33,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:33,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:33,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:34,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:34,836 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:35,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:36,249 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:36,272 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:36,837 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:37,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:38,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:39,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:40,275 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:41,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:41,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:42,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:43,273 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:44,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:44,839 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:45,290 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:46,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:46,945 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:47,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:47,901 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:56:48,241 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:56:48,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:56:48,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:49,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:50,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:51,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:52,253 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:52,282 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:52,842 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:53,289 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:54,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:55,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:56,283 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:57,254 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:56:57,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:58,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:56:58,844 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:56:59,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:00,284 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:01,291 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:01,541 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:57:01,541 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:57:01,541 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 with policy now +2025-01-07 20:57:01,542 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:01,542 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:01,542 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:01,769 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/hrcj4vaj-media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:01,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:57:02,118 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ecjsfkzd-media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:57:02,297 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:02,563 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:02,845 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:03,244 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:03,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:03,303 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:04,296 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:05,313 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:06,295 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:06,846 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:07,299 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:08,111 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:08,294 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:09,299 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:10,301 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:10,848 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:11,308 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:12,304 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:13,263 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:13,315 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:14,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:15,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:16,311 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:17,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:17,500 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:17,500 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:17,500 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:17,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:17,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:57:17,901 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:57:18,246 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:18,246 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:18,314 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:18,386 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:18,850 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:19,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:20,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:20,851 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:21,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:22,318 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:22,852 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:23,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:24,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:24,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:24,853 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:25,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:26,319 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:27,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:28,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:29,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:30,264 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:30,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:31,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:32,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:33,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:33,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:33,320 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:34,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:35,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:35,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:36,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:37,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:38,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:39,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:40,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:41,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:41,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:42,321 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:43,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:44,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:45,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:46,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:47,271 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:47,322 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:47,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:57:48,242 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:57:48,242 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:57:48,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:49,323 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:50,334 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:50,861 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:50,949 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:57:50,949 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 with policy now +2025-01-07 20:57:50,952 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:57:50,953 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:57:50,953 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:57:50,953 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:57:51,171 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/jfbqzpo7-media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:51,328 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:51,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:57:52,148 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ruz5h453-media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:57:52,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:52,862 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:53,005 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:53,334 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:54,335 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:55,333 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:56,331 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:56,863 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:57,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:58,281 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:57:58,333 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:57:58,864 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:57:59,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:00,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:01,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:02,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:03,248 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:03,249 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:03,336 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:03,370 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:04,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:05,341 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:06,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:07,337 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:08,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:09,285 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:09,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:10,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:10,566 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:10,566 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:10,567 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:10,868 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:10,869 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:11,132 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/gzq3p68q-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:11,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:12,347 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:12,869 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:13,341 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:14,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:14,347 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:15,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:16,342 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:16,871 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:17,346 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:17,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:18,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:18,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:18,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:18,871 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:19,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:20,313 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:20,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:21,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:22,343 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:23,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:24,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:25,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:26,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:26,344 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:27,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:28,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:29,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:30,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:31,274 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:31,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:32,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:33,243 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:33,243 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:33,345 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:34,346 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:35,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:35,647 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:35,647 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 with policy now +2025-01-07 20:58:35,657 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:35,657 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:35,657 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:35,658 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:35,877 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:35,877 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:35,878 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval +2025-01-07 20:58:35,900 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/5epie9ok-media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:36,165 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/4j3vtuu9-media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:58:36,349 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:36,683 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:36,878 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:37,351 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:38,356 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:39,358 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:40,350 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:40,879 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:41,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:42,136 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:42,458 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:43,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:44,451 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:44,880 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:45,451 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:46,449 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:47,292 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:47,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:47,902 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:48,250 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:58:48,250 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:58:48,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:49,457 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:50,466 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:51,306 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:58:51,306 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:58:51,456 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:51,567 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:51,827 INFO wandb-upload_1:2233136 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9iely7im +2025-01-07 20:58:51,883 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media +2025-01-07 20:58:51,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table +2025-01-07 20:58:51,885 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval +2025-01-07 20:58:52,087 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:58:52,096 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:58:52,198 INFO SenderThread:2233136 [sender.py:send_request_log_artifact():1525] logged artifact run-rgxendfg-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQzMjU3MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0NDc0Mw==', 'latestArtifact': None}} +2025-01-07 20:58:52,198 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:52,198 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-07 20:58:52,199 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:52,199 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,213 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,214 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send():379] send: telemetry +2025-01-07 20:58:52,214 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,230 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,230 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,244 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,244 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,244 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,245 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,255 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,255 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,266 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,267 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,279 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,279 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,280 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,280 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,290 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,290 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,309 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,309 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,309 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:52,325 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,325 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:52,326 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:52,326 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:58:52,391 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/gehzqjyy-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:52,454 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:52,653 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/kkz5ua2t-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:52,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval +2025-01-07 20:58:53,038 INFO wandb-upload_2:2233136 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdy65qdge +2025-01-07 20:58:53,172 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/pcbp8hgv-media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:53,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:53,543 INFO SenderThread:2233136 [sender.py:send_request_log_artifact():1525] logged artifact run-rgxendfg-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQzMjU5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTg0NDc0Ng==', 'latestArtifact': None}} +2025-01-07 20:58:53,543 DEBUG SenderThread:2233136 [sender.py:send():379] send: files +2025-01-07 20:58:53,544 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json with policy now +2025-01-07 20:58:53,544 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:58:53,544 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:58:53,544 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:53,805 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/ahy4g0j3-media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:58:53,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:54,456 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:54,884 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:58:55,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:56,455 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:58:57,427 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:58:57,473 DEBUG SenderThread:2233136 [sender.py:send():379] send: exit +2025-01-07 20:58:57,473 INFO SenderThread:2233136 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_exit():588] handling runtime: 249 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_exit():594] send defer +2025-01-07 20:58:57,474 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,474 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:58:57,474 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:58:57,474 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:58:57,474 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,474 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:58:57,475 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,475 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:58:57,475 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:58:57,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:58:57,475 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:58:57,475 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:58:57,475 DEBUG SystemMonitor:2233136 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:58:57,476 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:58:57,528 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:58:57,528 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:58:57,529 INFO HandlerThread:2233136 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:58:57,529 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,529 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:58:57,529 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:58:57,529 DEBUG SenderThread:2233136 [sender.py:send():379] send: stats +2025-01-07 20:58:57,529 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,529 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:58:57,530 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:58:57,530 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:58:57,530 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:58:57,530 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:58:57,530 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:58:57,886 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:58:58,886 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:59:03,626 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,627 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:59:03,627 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:03,627 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send():379] send: history +2025-01-07 20:59:03,627 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:59:03,627 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:59:03,629 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:59:03,629 DEBUG SenderThread:2233136 [sender.py:send():379] send: summary +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:59:03,630 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:59:03,630 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,630 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:59:03,630 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,630 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:59:03,634 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:59:03,770 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:59:03,770 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:03,770 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:59:03,770 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:03,770 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:59:03,888 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:59:03,888 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:59:04,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:05,727 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:59:05,727 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:05,728 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:05,728 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:59:05,728 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:05,728 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:59:05,728 INFO SenderThread:2233136 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:59:05,730 INFO SenderThread:2233136 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:59:05,732 INFO SenderThread:2233136 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:59:05,747 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:59:05,747 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:05,747 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:59:05,748 DEBUG SenderThread:2233136 [sender.py:send():379] send: artifact +2025-01-07 20:59:05,889 INFO Thread-12 :2233136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:59:06,021 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:59:06,021 INFO SenderThread:2233136 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:59:06,475 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml config.yaml +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json wandb-summary.json +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt requirements.txt +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:59:06,890 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log output.log +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/diff.patch diff.patch +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 media/videos/eval/2_eval_2_801e83780d634b6f5c3a.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 media/videos/eval/3_eval_3_5db1b98db4e00caa96ce.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 media/videos/eval/0_eval_0_c99588144472d6a23cee.mp4 +2025-01-07 20:59:06,894 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 media/videos/eval/1_eval_1_a3ac1d3949294c0444e4.mp4 +2025-01-07 20:59:06,895 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 media/videos/eval/4_eval_4_b54e419694f010e143b0.mp4 +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json media/table/eval/total_success_rate_ci_6_26598ad0ab173325629d.table.json +2025-01-07 20:59:06,899 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:59:06,899 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:06,900 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:06,900 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:59:06,900 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:06,900 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:59:06,900 INFO SenderThread:2233136 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:59:07,060 INFO wandb-upload_2:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/wandb-summary.json +2025-01-07 20:59:07,084 INFO wandb-upload_1:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/config.yaml +2025-01-07 20:59:07,120 INFO wandb-upload_0:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/requirements.txt +2025-01-07 20:59:07,125 INFO wandb-upload_4:2233136 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/files/output.log +2025-01-07 20:59:07,154 INFO wandb-upload_3:2233136 [upload_job.py:push():130] Uploaded file /tmp/tmpx17q8pktwandb/oj037jl0-events.out.tfevents.1736301288.leonmkim-ROG-Strix-G15CS-G15CS.2232948.0 +2025-01-07 20:59:07,354 INFO Thread-11 (_thread_body):2233136 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:59:07,354 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,354 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:59:07,355 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,355 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:59:07,355 INFO SenderThread:2233136 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:59:07,356 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,356 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:59:07,356 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:59:07,356 INFO SenderThread:2233136 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:59:07,476 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,536 INFO SenderThread:2233136 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:59:07,536 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:59:07,536 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,536 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,536 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send():379] send: final +2025-01-07 20:59:07,537 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:59:07,537 INFO HandlerThread:2233136 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send():379] send: footer +2025-01-07 20:59:07,537 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: defer +2025-01-07 20:59:07,537 INFO SenderThread:2233136 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:59:07,538 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,538 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,538 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:59:07,539 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:59:07,539 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:59:07,541 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:59:07,594 DEBUG SenderThread:2233136 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:59:07,594 INFO MainThread:2233136 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:59:07,595 DEBUG HandlerThread:2233136 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:59:07,595 INFO HandlerThread:2233136 [handler.py:finish():882] shutting down handler +2025-01-07 20:59:08,539 INFO WriterThread:2233136 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [sender.py:finish():1615] shutting down sender +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:59:08,594 INFO SenderThread:2233136 [file_pusher.py:join():175] waiting for file pusher diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug.log b/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e2872a7314d1662aeee5f17a3cd832b790c265e0 --- /dev/null +++ b/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Configure stats pid to 2232948 +2025-01-07 20:54:47,550 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/settings +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:54:47,551 INFO MainThread:2232948 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug.log +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/wandb/run-20250107_205447-rgxendfg/logs/debug-internal.log +2025-01-07 20:54:47,552 INFO MainThread:2232948 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg'} +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():619] starting backend +2025-01-07 20:54:47,553 INFO MainThread:2232948 [wandb_init.py:init():623] setting up manager +2025-01-07 20:54:47,563 INFO MainThread:2232948 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:54:47,564 INFO MainThread:2232948 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:54:47,577 INFO MainThread:2232948 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:54:47,585 INFO MainThread:2232948 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:54:47,816 INFO MainThread:2232948 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:54:47,886 INFO MainThread:2232948 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:54:47,886 INFO MainThread:2232948 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:54:48,239 INFO MainThread:2232948 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:54:48,241 INFO MainThread:2232948 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:54:48,241 INFO MainThread:2232948 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/rgxendfg/tb, True +2025-01-07 20:54:52,459 INFO MainThread:2232948 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:59:08,595 WARNING MsgRouterThr:2232948 [router.py:message_loop():77] message_loop has been closed diff --git a/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb b/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..920cd3f0593e8da635f1b527636c6f208fbf9cc5 Binary files /dev/null and b/rgxendfg/wandb/run-20250107_205447-rgxendfg/run-rgxendfg.wandb differ diff --git a/z6f2akbj/eval_robot.log b/z6f2akbj/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..a2938763441df65add011ce7544d15ee280e135e --- /dev/null +++ b/z6f2akbj/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:59:17,699][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:59:17,702][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( +