diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..40e859be2deb8ae933dbf032e6c4d6b25d6987df 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,76 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_15_2025-01-18-19-43-58.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_16_2025-01-18-19-44-51.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_2_2025-01-18-19-32-55.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_10_2025-01-18-19-39-16.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_11_2025-01-18-19-40-12.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_18_2025-01-18-19-46-47.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_12_2025-01-18-19-41-06.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_5_2025-01-18-19-35-10.bag filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_6_2025-01-18-19-36-04.bag filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/episode_rosbags/episode_3_2025-01-21-21-09-21.bag filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/episode_rosbags/episode_2_2025-01-21-21-08-37.bag filter=lfs diff=lfs merge=lfs -text +bn7ul4i2/episode_rosbags/episode_1_2025-01-21-21-07-52.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_14_2025-01-18-19-43-02.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_1_2025-01-18-19-32-05.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_17_2025-01-18-19-45-48.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_19_2025-01-18-19-47-45.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_13_2025-01-18-19-42-06.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_3_2025-01-18-19-33-41.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag filter=lfs diff=lfs merge=lfs -text +138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag filter=lfs diff=lfs merge=lfs -text +mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag filter=lfs diff=lfs merge=lfs -text diff --git a/138tidw7/.hydra/config.yaml b/138tidw7/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd7a0d9043cd058869b33cb503156d329d779fab --- /dev/null +++ b/138tidw7/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/138tidw7/.hydra/hydra.yaml b/138tidw7/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da269274d2f5634eb5a4643038a5970d7108a673 --- /dev/null +++ b/138tidw7/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/138tidw7/.hydra/overrides.yaml b/138tidw7/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/138tidw7/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/138tidw7/episode_rosbags/aligned_depth_to_color_K.npy b/138tidw7/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/138tidw7/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/138tidw7/episode_rosbags/cam_tf_world.npy b/138tidw7/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/138tidw7/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/138tidw7/episode_rosbags/color_K.npy b/138tidw7/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/138tidw7/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/138tidw7/episode_rosbags/depth_K.npy b/138tidw7/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/138tidw7/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/138tidw7/episode_rosbags/episode_10_2025-01-18-19-39-16.bag b/138tidw7/episode_rosbags/episode_10_2025-01-18-19-39-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..70761c64cc5d47acbbbfc1a41c202435a13e6a79 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_10_2025-01-18-19-39-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddd3371c6773ff4d4fd03596b613058932236d96e965cb654252ac3bd6bdf9e7 +size 1410759910 diff --git a/138tidw7/episode_rosbags/episode_11_2025-01-18-19-40-12.bag b/138tidw7/episode_rosbags/episode_11_2025-01-18-19-40-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..4556ce8a62b54adb7505f178ac3bc9de2322d1d3 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_11_2025-01-18-19-40-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61acdc8ed1e67391a499abb3449f12a4def31fd332bd0a0be58fc5d8ef58d4c7 +size 1410778565 diff --git a/138tidw7/episode_rosbags/episode_12_2025-01-18-19-41-06.bag b/138tidw7/episode_rosbags/episode_12_2025-01-18-19-41-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..f8c33786a9168ed8a9efe5e000ff7aec1eef120c --- /dev/null +++ b/138tidw7/episode_rosbags/episode_12_2025-01-18-19-41-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f960dd97054d8122823e0eff8b8f07b2e90524324a699a1c1959f8f91756fda5 +size 1409674308 diff --git a/138tidw7/episode_rosbags/episode_13_2025-01-18-19-42-06.bag b/138tidw7/episode_rosbags/episode_13_2025-01-18-19-42-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..b78bb1f1ccc4c775f131a9bfc42da60d0f66f689 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_13_2025-01-18-19-42-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96a24d6ca9c38eece5dabf59cd82e54199691012456e4df4b4bb13669b7156f +size 1416620288 diff --git a/138tidw7/episode_rosbags/episode_14_2025-01-18-19-43-02.bag b/138tidw7/episode_rosbags/episode_14_2025-01-18-19-43-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..b1fdee19cc9f06d6d3ad7c0847a764d1dea4f6d7 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_14_2025-01-18-19-43-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:380b4a0e52f801608e24ac513ba9ef2e51f25eb1c306af6724a117859aec59e5 +size 1407745740 diff --git a/138tidw7/episode_rosbags/episode_15_2025-01-18-19-43-58.bag b/138tidw7/episode_rosbags/episode_15_2025-01-18-19-43-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..925b7f37c382ef4c9cfd4b42f67443185ab16ace --- /dev/null +++ b/138tidw7/episode_rosbags/episode_15_2025-01-18-19-43-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e1841ff9df9160d4f288f76c1df1982450cb5faf22f1a8ce1b1e6788fda2f68 +size 1412451850 diff --git a/138tidw7/episode_rosbags/episode_16_2025-01-18-19-44-51.bag b/138tidw7/episode_rosbags/episode_16_2025-01-18-19-44-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..544fd77e62ab955da9e316556c50783d9250eff7 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_16_2025-01-18-19-44-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfba0de2b3d286370cd8b93d9eb122b5af52f28705fa18b953ec64964cba07a9 +size 1412873005 diff --git a/138tidw7/episode_rosbags/episode_17_2025-01-18-19-45-48.bag b/138tidw7/episode_rosbags/episode_17_2025-01-18-19-45-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..da66a6c542ab70ef0e309b2ca3aaab38f4264c1b --- /dev/null +++ b/138tidw7/episode_rosbags/episode_17_2025-01-18-19-45-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7da9c6c396736b68f2086ad978b45fb364771d29ccb3cdbf21e9fd948fefd6d +size 1411258296 diff --git a/138tidw7/episode_rosbags/episode_18_2025-01-18-19-46-47.bag b/138tidw7/episode_rosbags/episode_18_2025-01-18-19-46-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..e4a4e298ee953a8b451c3236b1f0e499a3a00eca --- /dev/null +++ b/138tidw7/episode_rosbags/episode_18_2025-01-18-19-46-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234022a6829c4c3e0071a340f1d35fc652ecff3f76476ee86b74b4b5f3d62618 +size 1417816560 diff --git a/138tidw7/episode_rosbags/episode_19_2025-01-18-19-47-45.bag b/138tidw7/episode_rosbags/episode_19_2025-01-18-19-47-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..34cf9efd29ca79ac5e51ca644dbf786884762eed --- /dev/null +++ b/138tidw7/episode_rosbags/episode_19_2025-01-18-19-47-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ac4a0ac3c5f8d4e8986dd0e078d55767c030660a6f93b0fdcba1f573e685a9 +size 1409867638 diff --git a/138tidw7/episode_rosbags/episode_1_2025-01-18-19-32-05.bag b/138tidw7/episode_rosbags/episode_1_2025-01-18-19-32-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..43cbd7a02275c147e46867769ea95bd2975c69e2 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_1_2025-01-18-19-32-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fe467be3621efd34e488c8543cb962e553fac6664bdb0583eae4828f12c10d7 +size 1409916872 diff --git a/138tidw7/episode_rosbags/episode_2_2025-01-18-19-32-55.bag b/138tidw7/episode_rosbags/episode_2_2025-01-18-19-32-55.bag new file mode 100644 index 0000000000000000000000000000000000000000..00a1f78ac6d51945df283fde60112be3ffa34829 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_2_2025-01-18-19-32-55.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7396bb0542f6e41079ac48c5675f2b2be5f30f0a605ad29a81f01c4d6aa74d42 +size 1411067620 diff --git a/138tidw7/episode_rosbags/episode_3_2025-01-18-19-33-41.bag b/138tidw7/episode_rosbags/episode_3_2025-01-18-19-33-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..519ae9c79f48b4f2a3480427327767c15ddd5e27 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_3_2025-01-18-19-33-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7570df82d4236bf2b948751eb64569e03c50843f5b046551947409eb228ad7c +size 1409215227 diff --git a/138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag b/138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..e6665e5be7faf31d64362858dfce34e6db1b3b33 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6190c3c091bf31cb49debc226a2e1cf7d3d659a9afe0508dacb82d6d66e10eb3 +size 1411204932 diff --git a/138tidw7/episode_rosbags/episode_5_2025-01-18-19-35-10.bag b/138tidw7/episode_rosbags/episode_5_2025-01-18-19-35-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..685589b35bf3ced098e75b3ef14f3eafbc8c39c8 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_5_2025-01-18-19-35-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80085fcb26b2cf8dc7823a2e74ab29cff80dd4436d198a346d5b5ff44463930e +size 1411353293 diff --git a/138tidw7/episode_rosbags/episode_6_2025-01-18-19-36-04.bag b/138tidw7/episode_rosbags/episode_6_2025-01-18-19-36-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..576bce432bd3f24349b9b5d5871a21096131d6d7 --- /dev/null +++ b/138tidw7/episode_rosbags/episode_6_2025-01-18-19-36-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a99f4e575ec881239a082c8593053a7c939cd6ef54a61c95eed3383dee3618 +size 1407181611 diff --git a/138tidw7/eval_robot.log b/138tidw7/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..568403e7841279ea9ac7d224b7a3969c9eba0554 --- /dev/null +++ b/138tidw7/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 19:30:29,570][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 19:30:29,573][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/138tidw7/eval_video/0_eval.mp4 b/138tidw7/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5a524fa702400ae0fe8fd95309174bd7aa60db2 --- /dev/null +++ b/138tidw7/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96591af35a54c45302d0a9395a677d03a00e4e3c32409560bff49f0868d21c7a +size 1111368 diff --git a/138tidw7/eval_video/10_eval.mp4 b/138tidw7/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f8d9a9f5a939836fc5492a52b8131f907004b49 --- /dev/null +++ b/138tidw7/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b221e9eaaa2afeafab23d9f82015bfca05527b928936906a55e5a9e06446ed +size 1379873 diff --git a/138tidw7/eval_video/11_eval.mp4 b/138tidw7/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3eb19ac955aa4562476ecd62267a99db16fb2fb8 --- /dev/null +++ b/138tidw7/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d95850d1df3e2b923407436043bd539b9210471db5dff5ae624216157c3095 +size 1448723 diff --git a/138tidw7/eval_video/12_eval.mp4 b/138tidw7/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c97379d5846507462e626be6035d2325093d9247 --- /dev/null +++ b/138tidw7/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b1f9ef1d691b07f19acb5f6dbd0edb5a56defa33558799db6d69e9167a70f5 +size 1499478 diff --git a/138tidw7/eval_video/13_eval.mp4 b/138tidw7/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a439658d866263626da781e8e0c17cc1f1a497f --- /dev/null +++ b/138tidw7/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a298f612d50a7b95e2210ba8a86026cb6debdb5f035ded03455ff2f7be1b933e +size 1453691 diff --git a/138tidw7/eval_video/14_eval.mp4 b/138tidw7/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2336253fd14f6791dbd67f5451255f1e20fe751b --- /dev/null +++ b/138tidw7/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da508fd72acfd8e8c94ec6cd36d8a75233551e542aa1e99546a31fee0e34f92f +size 1515972 diff --git a/138tidw7/eval_video/15_eval.mp4 b/138tidw7/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..204b0d1c73a1d5b669ad3e5ff3af1f425c530208 --- /dev/null +++ b/138tidw7/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f145ad74c481c07c43b6f16cbb657074336b35655bdd0e29efad994b960229 +size 1629682 diff --git a/138tidw7/eval_video/16_eval.mp4 b/138tidw7/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4b1d727fbb86c5b1f4f983ed07e87584d4b40f15 --- /dev/null +++ b/138tidw7/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4567bcc512ee0512f3ae490be445b8a9b07cd9ca027fe3726d8f4b23836ee34f +size 1619329 diff --git a/138tidw7/eval_video/17_eval.mp4 b/138tidw7/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4fad187a04303e93dbfe6c352b1ac00a8ef9cf5f --- /dev/null +++ b/138tidw7/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415f21e10d2661428c434b6830ac89b499e81e0e2ac28b80f4367e3d27fcb141 +size 1693625 diff --git a/138tidw7/eval_video/18_eval.mp4 b/138tidw7/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a0689c3e8b469193aa63bad424d473a5e694b75 --- /dev/null +++ b/138tidw7/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c84898aed318ecd95cd83b9d459daea4ca54b8963dedad94016c4ace6718e58 +size 1713057 diff --git a/138tidw7/eval_video/19_eval.mp4 b/138tidw7/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ca04b3f0ab4459ed29621eadc5e3750a5340813 --- /dev/null +++ b/138tidw7/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d5fa28d764eeaa07d1aec6e42f028f7e27173f4955f7b3bc8d442c5d180f5f +size 1480591 diff --git a/138tidw7/eval_video/1_eval.mp4 b/138tidw7/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76fe0aa1cae95f3ecec3e2ed8cffbc5238e4133a --- /dev/null +++ b/138tidw7/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0aac6515bc4a872c577de09d43dec9134d3a60ceedfb842c27ae3a77272c2a5 +size 1193043 diff --git a/138tidw7/eval_video/2_eval.mp4 b/138tidw7/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b623019bb7287ef8a554c263200329eaf0bc2b94 --- /dev/null +++ b/138tidw7/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcfa4207d555c0dc155d3bcf40a93b4285f85d8425c2686d714270bc309487ee +size 1211592 diff --git a/138tidw7/eval_video/3_eval.mp4 b/138tidw7/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9edd55caef748e7bfb0ec0630e51f2550793fb4 --- /dev/null +++ b/138tidw7/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6854d6b9b0ee461bdf8cf13ae97bc2a808f179bc878f37657dba33fd07640c29 +size 1157029 diff --git a/138tidw7/eval_video/4_eval.mp4 b/138tidw7/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5d7818f6554484e0684ea974d4abca11a3b2519 --- /dev/null +++ b/138tidw7/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573f9d77908239cd99f44510bb179b8515f548ec362d12ceccdd2a47e32d1188 +size 1220051 diff --git a/138tidw7/eval_video/5_eval.mp4 b/138tidw7/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb81252d15ee87c5b254657560e04bc249a8d8bf --- /dev/null +++ b/138tidw7/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6b889b8499cd4be78a68284395f6b7d7bcab408235e139ea67590b4af6cd73 +size 1123543 diff --git a/138tidw7/eval_video/6_eval.mp4 b/138tidw7/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..079d7ef73fa777c605e9d2cce8cd7ae91c61b867 --- /dev/null +++ b/138tidw7/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d85fdaf1b462a9c0774327705488b9c5ab10fc274b3c27ec53c7f18a4d56263 +size 1199797 diff --git a/138tidw7/eval_video/7_eval.mp4 b/138tidw7/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af0e02cf7d327f8895cdef8a0790c40a1c8b399d --- /dev/null +++ b/138tidw7/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bd94bd9f2e25c7a138f737e6e83ed926b3c8a185481378cdfdded93f5e3dfe +size 1296513 diff --git a/138tidw7/eval_video/8_eval.mp4 b/138tidw7/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e0370b78029d97882755908942cc6cd7320854a --- /dev/null +++ b/138tidw7/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad80c418d3b655ed57abd05bd86565ae377f9fb98ccfaee5a538d56a4bd2253 +size 1348396 diff --git a/138tidw7/eval_video/9_eval.mp4 b/138tidw7/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65f1cf8b62d35d89c1f5faafce297f7a1a2920a6 --- /dev/null +++ b/138tidw7/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc9e9466280c44c4ea9fe175a9c581ee927269612a62b0329244346f94b32e6 +size 1234297 diff --git a/138tidw7/tb/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 b/138tidw7/tb/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 new file mode 100644 index 0000000000000000000000000000000000000000..50ced6a69beae199e7cef24ddb0a61f454cf7bb6 --- /dev/null +++ b/138tidw7/tb/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0747b37bf0e8c8e3a393dc1b06aa2b666dfde7598108f70e5d054247738187d3 +size 22784 diff --git a/138tidw7/wandb/debug-internal.log b/138tidw7/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..faa46b32003400abf76ed956126f58382d3d5215 --- /dev/null +++ b/138tidw7/wandb/debug-internal.log @@ -0,0 +1,2321 @@ +2025-01-18 19:30:35,313 INFO StreamThr :2561591 [internal.py:wandb_internal():85] W&B internal server running at pid: 2561591, started at: 2025-01-18 19:30:35.312670 +2025-01-18 19:30:35,314 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:30:35,324 INFO WriterThread:2561591 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb +2025-01-18 19:30:35,325 DEBUG SenderThread:2561591 [sender.py:send():379] send: header +2025-01-18 19:30:35,334 DEBUG SenderThread:2561591 [sender.py:send():379] send: run +2025-01-18 19:30:35,573 INFO SenderThread:2561591 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files +2025-01-18 19:30:35,573 INFO SenderThread:2561591 [sender.py:_start_run_threads():1188] run started: 138tidw7 with start time 1737246635.313005 +2025-01-18 19:30:35,592 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:30:35,592 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:30:35,662 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:30:35,667 DEBUG HandlerThread:2561591 [system_info.py:__init__():26] System info init +2025-01-18 19:30:35,667 DEBUG HandlerThread:2561591 [system_info.py:__init__():41] System info init done +2025-01-18 19:30:35,667 INFO HandlerThread:2561591 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:30:35,667 INFO SystemMonitor:2561591 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:30:35,667 INFO HandlerThread:2561591 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:30:35,668 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:30:35,668 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:30:35,680 DEBUG HandlerThread:2561591 [system_info.py:probe():152] Probing system +2025-01-18 19:30:35,682 DEBUG HandlerThread:2561591 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:30:35,686 DEBUG HandlerThread:2561591 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:30:35,686 DEBUG HandlerThread:2561591 [system_info.py:probe():200] Probing system done +2025-01-18 19:30:35,687 DEBUG HandlerThread:2561591 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:30:35.680913', 'startedAt': '2025-01-19T00:30:35.299465', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.9246249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4520.276, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4515.837, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 550.1006927490234}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:30:35,687 INFO HandlerThread:2561591 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:30:35,687 INFO HandlerThread:2561591 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:30:35,687 DEBUG HandlerThread:2561591 [system_info.py:_save_code():44] Saving code +2025-01-18 19:30:35,692 DEBUG HandlerThread:2561591 [system_info.py:_save_code():67] Saving code done +2025-01-18 19:30:35,693 DEBUG HandlerThread:2561591 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 19:30:35,787 DEBUG HandlerThread:2561591 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 19:30:35,788 INFO HandlerThread:2561591 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:30:35,791 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 19:30:35,792 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 19:30:36,024 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:30:36,024 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:30:36,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:30:36,027 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:30:36,049 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/2l4l5vgf-diff.patch +2025-01-18 19:30:36,051 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/wxd025z5-wandb-metadata.json +2025-01-18 19:30:36,052 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/a313lqel-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:30:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:36,069 INFO HandlerThread:2561591 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/tb" + save: true +} +control { + relay_id: "d6a45ccaf7c14b3aa590c38ecc0ce170" +} +_info { + stream_id: "138tidw7" +} + +2025-01-18 19:30:36,069 DEBUG HandlerThread:2561591 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:30:36,084 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/ofmwlwa8-code/FISH/eval_robot.py +2025-01-18 19:30:36,168 DEBUG SenderThread:2561591 [sender.py:send():379] send: telemetry +2025-01-18 19:30:36,168 DEBUG SenderThread:2561591 [sender.py:send():379] send: tbrecord +2025-01-18 19:30:36,169 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:30:36,169 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 with policy live +2025-01-18 19:30:36,354 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/sr3ob2h2-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json +2025-01-18 19:30:36,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code +2025-01-18 19:30:36,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH +2025-01-18 19:30:37,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:38,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:38,574 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:39,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:40,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:41,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:41,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:42,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:43,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:44,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:44,487 DEBUG SenderThread:2561591 [sender.py:send():379] send: config +2025-01-18 19:30:45,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:46,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:46,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:46,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:47,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:48,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:49,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:50,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:50,577 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:51,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:30:51,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:30:51,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:51,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:52,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:53,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:54,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:54,579 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:55,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:56,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:57,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:57,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:58,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:59,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:00,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:01,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:02,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:02,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:03,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:04,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:05,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:06,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:06,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:06,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:07,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:07,133 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:07,582 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:31:08,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:09,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:10,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:11,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:12,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:13,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:13,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:14,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:15,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:16,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:17,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:18,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:18,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:19,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:20,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:20,586 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:21,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:21,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:21,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:22,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:23,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:23,097 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:24,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:24,587 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:25,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:26,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:27,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:28,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:29,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:29,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:30,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:31,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:32,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:33,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:34,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:34,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:35,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:35,670 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:31:35,670 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:31:36,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:36,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:37,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:38,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:39,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:39,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:40,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:41,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:42,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:43,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:44,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:45,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:45,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:46,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:47,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:48,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:49,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:49,734 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:31:49,734 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:31:49,736 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 with policy now +2025-01-18 19:31:49,736 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:31:49,737 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:31:49,737 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:31:49,992 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/x2zq8vpn-media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:50,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:50,594 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:50,595 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:31:50,595 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:31:50,743 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:50,868 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jz2zp5au-media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:51,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:51,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:52,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:53,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:54,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:54,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:55,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:56,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:56,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:56,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:57,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:58,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:59,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:00,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:01,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:01,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:01,598 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:01,849 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/nxakqe0x-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:02,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:02,599 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:03,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:04,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:04,599 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:05,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:05,671 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:32:06,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:06,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:06,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:06,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:06,600 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:07,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:08,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:08,601 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:09,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:10,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:11,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:12,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:12,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:13,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:14,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:15,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:16,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:17,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:18,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:18,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:19,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:20,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:21,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:21,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:21,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:22,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:22,606 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:23,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:23,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:24,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:25,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:26,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:27,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:28,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:28,599 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:29,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:30,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:30,608 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:31,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:32,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:33,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:33,969 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:34,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:34,217 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:32:34,218 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 with policy now +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:32:34,218 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:32:34,466 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/kmrnxouo-media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:32:34,884 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/ih4zm23x-media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:35,048 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:35,671 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:32:36,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:36,027 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:36,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:37,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:38,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:38,611 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:39,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:39,785 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:40,054 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:41,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:42,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:42,612 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:43,056 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:44,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:45,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:45,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:46,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:47,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:48,063 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:49,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:50,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:51,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:51,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:51,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:52,064 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:52,510 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:32:52,510 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:32:52,511 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:53,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:54,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:54,617 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:55,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:56,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:56,547 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:56,618 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:57,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:58,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:58,618 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:59,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:00,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:01,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:02,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:02,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:03,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:04,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:05,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:05,672 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:33:06,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:06,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:06,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:07,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:07,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:08,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:09,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:10,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:11,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:12,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:13,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:13,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:14,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:15,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:16,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:17,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:18,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:18,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:19,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:20,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:21,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:21,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:21,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:22,071 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:22,626 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:23,071 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:23,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:24,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:24,942 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:33:24,943 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 with policy now +2025-01-18 19:33:24,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:33:24,946 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:33:24,946 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:33:24,946 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:33:25,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:25,205 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/1r0tvj7i-media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:25,627 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:33:25,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:25,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:33:25,891 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3wqpggxx-media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:26,079 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:26,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:27,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:28,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:28,286 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:29,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:30,078 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:30,629 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:31,105 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:32,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:32,630 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:33,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:34,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:34,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:35,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:35,672 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:33:36,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:36,034 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:36,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:37,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:37,631 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:33:37,876 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3ktxe6xs-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:33:38,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:38,345 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:33:38,345 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:33:38,345 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:33:38,632 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:33:38,632 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:39,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:39,601 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:40,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:40,633 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:41,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:42,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:42,633 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:43,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:44,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:44,634 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:44,948 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:45,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:46,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:47,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:48,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:49,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:50,046 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:50,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:51,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:51,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:51,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:52,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:53,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:54,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:55,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:55,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:56,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:57,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:58,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:59,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:00,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:01,045 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:01,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:02,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:03,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:04,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:05,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:05,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:34:06,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:06,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:06,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:06,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:07,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:08,095 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:09,095 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:10,104 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:10,637 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:34:10,637 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 with policy now +2025-01-18 19:34:10,641 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:34:10,642 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:10,642 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:10,642 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:34:10,925 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/l5wz1cf1-media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:11,117 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:11,195 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/868y88g3-media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:12,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:12,116 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:12,644 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:13,116 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:14,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:15,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:16,118 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:16,645 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:17,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:17,993 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:18,109 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:18,646 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:19,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:20,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:21,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:21,032 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:21,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:22,115 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:23,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:23,142 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:24,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:25,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:25,402 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:25,403 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:25,403 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:25,648 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:25,648 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:34:26,123 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:26,649 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:27,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:28,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:28,497 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:28,649 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:29,130 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:30,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:30,650 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:31,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:32,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:32,651 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:33,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:34,051 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:34,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:35,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:35,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:34:36,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:36,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:36,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:37,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:38,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:39,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:39,174 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:40,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:41,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:42,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:43,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:44,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:45,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:45,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:46,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:47,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:48,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:49,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:50,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:50,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:51,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:51,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:52,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:53,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:54,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:55,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:55,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:56,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:57,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:58,137 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:58,385 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:34:58,385 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 with policy now +2025-01-18 19:34:58,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:34:58,388 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:58,388 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:58,389 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:58,629 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zwzzhutz-media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:58,660 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:58,660 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:58,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:58,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:34:58,916 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/g1xudixv-media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:59,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:00,146 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:00,426 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:00,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:01,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:02,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:02,662 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:03,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:04,147 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:05,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:05,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:05,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:35:06,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:06,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:06,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:06,663 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:06,663 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:35:06,900 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/a5qhx72n-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:35:07,148 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:07,462 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:35:07,462 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:35:07,464 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:35:07,664 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:35:08,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:08,664 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:09,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:10,159 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:10,505 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:11,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:12,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:12,665 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:13,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:14,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:15,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:16,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:16,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:17,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:18,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:19,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:20,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:21,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:21,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:21,139 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:21,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:22,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:23,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:24,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:25,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:26,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:26,670 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:27,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:27,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:28,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:29,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:30,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:31,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:32,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:32,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:32,673 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:33,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:34,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:35,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:35,674 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:35:36,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:36,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:36,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:37,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:38,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:38,157 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:39,159 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:39,184 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:35:39,184 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 with policy now +2025-01-18 19:35:39,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:35:39,225 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:35:39,226 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:35:39,226 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:35:39,431 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/g79dgf1j-media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:35:39,920 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/uni7awtz-media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:40,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:40,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:41,172 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:42,173 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:43,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:43,669 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:44,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:44,678 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:45,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:46,203 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:46,679 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:47,177 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:48,186 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:49,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:49,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:50,184 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:51,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:51,036 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:51,196 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:52,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:53,190 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:54,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:54,196 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:55,202 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:56,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:57,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:58,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:59,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:00,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:00,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:01,219 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:01,529 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:01,530 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:01,530 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:01,684 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:01,684 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:02,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:02,685 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:03,228 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:04,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:05,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:05,674 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:36:05,675 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:06,050 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:06,050 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:06,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:06,686 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:07,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:08,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:08,687 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:09,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:10,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:11,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:11,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:12,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:13,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:14,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:15,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:16,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:16,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:17,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:18,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:19,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:20,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:20,691 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:21,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:21,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:21,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:21,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:22,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:23,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:24,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:24,693 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:25,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:26,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:26,693 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:27,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:27,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:28,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:29,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:30,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:31,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:32,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:32,902 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:33,235 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:33,987 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:36:33,987 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 with policy now +2025-01-18 19:36:33,996 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:36:33,996 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:33,996 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:33,997 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:34,219 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:34,238 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/65xpujy0-media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:34,696 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:36:34,986 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/qcrnyxba-media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:35,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:35,675 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:36:36,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:36,058 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:36,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:37,233 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:38,238 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:38,277 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:38,698 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:39,234 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:40,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:40,578 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:40,579 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:40,579 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:40,699 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:40,699 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:40,945 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3p1t47or-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:41,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:42,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:42,700 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:43,240 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:43,621 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:44,254 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:45,258 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:46,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:46,701 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:47,261 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:48,262 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:48,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:49,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:50,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:50,702 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:51,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:51,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:51,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:52,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:53,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:54,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:54,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:54,704 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:55,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:56,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:57,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:58,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:59,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:00,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:00,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:01,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:02,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:03,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:04,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:05,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:05,675 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:37:05,676 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:06,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:06,037 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:06,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:07,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:08,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:09,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:10,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:11,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:11,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:12,254 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:13,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:14,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:15,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:16,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:16,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:17,271 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:17,696 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:37:17,696 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 with policy now +2025-01-18 19:37:17,700 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:37:17,701 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:37:17,701 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:37:17,701 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:37:17,985 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/m4je42ln-media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:18,272 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/8hsb26w0-media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:18,285 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:18,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:19,268 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:20,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:21,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:21,059 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:21,189 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:21,284 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:22,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:22,714 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:23,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:24,275 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:25,276 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:26,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:26,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:26,635 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:37:26,635 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:37:26,635 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:37:27,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:28,292 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:28,717 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:29,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:30,321 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:31,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:32,104 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:32,306 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:32,718 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:33,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:34,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:35,312 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:35,676 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:37:36,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:36,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:36,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:36,720 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:37,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:37,752 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:38,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:38,721 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:39,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:40,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:41,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:42,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:43,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:43,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:44,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:44,723 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:45,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:46,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:47,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:48,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:49,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:49,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:50,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:51,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:51,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:51,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:52,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:53,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:54,140 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:54,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:55,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:56,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:57,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:58,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:59,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:00,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:00,306 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:01,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:02,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:03,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:04,319 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:04,538 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:38:04,538 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 with policy now +2025-01-18 19:38:04,545 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:38:04,545 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:38:04,545 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:38:04,545 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:04,732 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:38:04,811 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7azmvai9-media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:05,075 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/mugxa2fy-media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:05,324 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:05,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:05,676 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:38:06,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:06,081 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:06,328 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:06,732 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:07,342 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:08,344 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:08,733 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:09,350 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:10,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:11,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:11,332 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:12,339 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:12,735 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:13,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:14,357 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:15,383 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:16,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:16,348 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:17,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:18,359 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:19,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:20,362 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:21,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:21,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:21,125 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:21,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:22,354 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:23,355 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:24,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:25,378 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:26,385 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:27,108 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:27,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:28,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:28,712 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:38:28,712 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:38:28,712 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:38:28,977 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/4pcu0qo7-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:38:29,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:30,397 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:30,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:31,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:32,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:32,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:33,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:34,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:34,743 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:35,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:35,677 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:38:36,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:36,043 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:36,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:37,182 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:37,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:38,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:39,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:40,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:41,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:42,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:43,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:43,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:44,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:45,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:46,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:47,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:48,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:48,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:49,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:50,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:51,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:51,043 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:51,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:52,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:53,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:53,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:54,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:55,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:56,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:57,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:58,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:58,470 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:59,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:00,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:00,753 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:01,377 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:01,622 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:39:01,622 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 with policy now +2025-01-18 19:39:01,643 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:39:01,644 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:01,644 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:01,646 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:01,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:01,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:01,755 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:39:01,878 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jtoz3yny-media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:02,133 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/i0cb5oxv-media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:02,381 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:02,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:03,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:03,708 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:04,402 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:05,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:05,677 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:39:06,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:06,048 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:06,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:06,756 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:07,410 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:08,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:08,757 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:09,221 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:09,412 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:10,407 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:11,399 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:12,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:12,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:12,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:39:12,764 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:12,764 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:12,765 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:13,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:13,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:14,411 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:14,760 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:15,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:15,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:16,412 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:16,760 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:17,423 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:18,441 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:19,442 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:20,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:20,448 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:21,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:21,044 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:21,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:22,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:22,762 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:23,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:24,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:25,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:25,878 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:26,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:27,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:28,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:29,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:30,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:31,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:31,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:32,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:33,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:34,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:35,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:35,678 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:39:36,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:36,044 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:36,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:36,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:37,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:38,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:38,769 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:39,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:40,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:41,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:41,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:42,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:42,771 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:43,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:44,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:45,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:46,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:47,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:47,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:48,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:49,448 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:49,719 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:39:49,719 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 with policy now +2025-01-18 19:39:49,747 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:39:49,747 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:49,748 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:49,748 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:49,774 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:49,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:49,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:39:49,987 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/xbabfegw-media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:50,242 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zolk9upm-media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:50,472 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:50,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:51,075 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:51,075 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:51,506 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:52,144 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:52,481 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:53,478 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:54,478 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:54,777 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:55,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:56,474 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:57,469 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:57,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:58,498 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:58,780 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:59,494 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:00,510 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:01,527 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:02,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:03,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:03,542 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:04,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:05,544 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:05,678 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:40:06,086 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:06,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:06,530 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:07,530 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:08,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:08,528 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:08,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:08,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:40:08,833 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:40:08,833 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:40:08,836 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:40:09,033 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/f48blnw7-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:40:09,537 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:09,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:40:10,535 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:10,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:11,543 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:12,563 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:13,536 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:13,617 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:14,566 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:14,786 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:15,617 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:16,597 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:17,612 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:18,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:19,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:19,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:20,585 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:20,788 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:21,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:21,047 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:21,585 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:22,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:22,789 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:23,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:24,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:24,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:25,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:26,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:27,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:28,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:29,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:30,148 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:30,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:31,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:32,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:33,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:34,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:35,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:35,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:35,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:40:36,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:36,047 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:36,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:37,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:38,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:39,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:40,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:40,595 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:41,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:42,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:43,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:44,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:45,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:46,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:46,592 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:47,603 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:48,354 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:40:48,354 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 with policy now +2025-01-18 19:40:48,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:40:48,363 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:40:48,363 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:40:48,363 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:40:48,614 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:48,616 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/z9ypi00b-media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:48,799 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:40:48,799 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:48,800 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:48,800 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:40:49,072 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/n10zum27-media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:49,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:50,603 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:50,803 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:51,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:51,065 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:51,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:51,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:52,630 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:52,804 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:53,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:54,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:55,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:56,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:56,806 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:57,189 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:57,646 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:58,650 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:59,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:00,674 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:01,644 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:02,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:02,654 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:02,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:02,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:41:02,896 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:41:02,897 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:41:02,897 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:41:03,676 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:03,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:41:04,663 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:04,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:05,655 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:05,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:41:06,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:06,058 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:06,659 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:07,488 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:07,665 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:08,654 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:08,811 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:09,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:10,685 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:11,699 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:12,683 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:13,181 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:13,685 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:14,704 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:14,813 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:15,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:16,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:16,814 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:17,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:18,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:19,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:19,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:20,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:21,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:21,049 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:21,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:22,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:23,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:24,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:25,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:25,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:26,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:27,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:28,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:29,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:30,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:31,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:31,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:32,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:33,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:34,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:35,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:41:35,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:36,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:36,049 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:36,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:36,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:37,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:38,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:39,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:40,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:41,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:41,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:42,698 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:42,824 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:42,866 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:41:42,866 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 with policy now +2025-01-18 19:41:42,878 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:41:42,878 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:41:42,878 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:41:42,879 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:41:43,121 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7uoavehf-media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:43,703 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:41:44,124 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/k57q4cig-media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:44,726 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:44,826 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:45,729 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:46,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:46,728 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:47,720 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:48,742 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:48,828 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:49,713 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:50,722 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:51,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:51,052 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:51,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:51,721 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:51,829 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:52,718 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:53,727 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:54,735 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:55,736 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:56,754 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:57,208 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:57,743 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:58,739 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:59,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:00,788 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:01,737 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:02,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:02,756 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:02,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:02,971 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:02,971 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:02,972 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:03,738 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:03,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:03,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:04,750 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:05,680 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:42:05,772 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:05,835 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:06,063 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:06,064 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:06,759 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:07,621 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:07,780 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:08,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:09,774 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:09,837 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:10,792 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:11,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:12,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:13,203 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:13,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:14,791 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:15,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:15,839 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:16,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:17,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:17,840 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:18,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:19,194 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:19,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:20,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:21,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:21,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:21,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:21,842 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:22,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:23,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:23,843 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:24,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:24,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:25,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:26,780 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:27,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:28,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:29,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:29,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:30,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:31,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:31,846 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:32,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:33,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:34,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:35,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:35,680 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:42:35,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:36,056 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:36,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:36,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:37,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:38,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:39,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:40,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:41,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:41,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:42,793 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:42,980 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:42:42,980 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 with policy now +2025-01-18 19:42:42,989 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:42:42,990 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:42,990 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:42,990 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:43,229 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/doq2mkg2-media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:43,823 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:42:44,136 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7z0vlu8s-media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:44,808 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:44,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:45,819 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:45,852 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:46,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:46,805 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:47,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:47,852 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:48,840 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:49,808 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:49,853 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:50,838 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:51,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:51,104 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:51,230 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:51,833 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:51,854 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:52,837 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:53,833 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:54,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:55,840 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:56,856 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:57,228 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:57,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:58,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:58,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:59,040 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:59,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:59,040 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:59,118 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7jlg2hev-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:59,856 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:59,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:59,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:00,907 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:01,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:01,859 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:02,751 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:02,855 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:03,861 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:04,893 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:05,681 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:43:05,860 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:05,907 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:06,085 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:06,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:06,892 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:07,893 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:08,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:08,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:09,947 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:10,928 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:11,927 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:12,913 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:13,491 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:13,864 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:13,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:14,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:15,864 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:15,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:16,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:17,865 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:17,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:18,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:19,361 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:19,916 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:20,916 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:21,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:21,060 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:21,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:22,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:23,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:24,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:25,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:25,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:26,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:27,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:28,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:29,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:30,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:31,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:31,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:32,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:33,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:34,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:35,681 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:43:35,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:36,061 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:36,061 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:36,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:36,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:37,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:38,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:39,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:40,932 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:41,217 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:41,510 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:43:41,510 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 with policy now +2025-01-18 19:43:41,558 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:43:41,559 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:43:41,559 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:43:41,559 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:43:41,757 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/vks9ezhq-media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:41,875 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:43:41,935 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:42,143 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jsu0qqf7-media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:42,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:42,930 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:43,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:43,936 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:44,946 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:45,933 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:46,227 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:46,936 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:47,878 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:47,944 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:48,930 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:49,879 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:49,970 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:50,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:51,087 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:51,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:51,259 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:51,955 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:52,953 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:53,959 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:54,105 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:43:54,105 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:43:54,105 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:43:54,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:43:54,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:43:54,954 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:55,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:55,948 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:56,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:56,950 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:57,882 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:57,959 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:58,965 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:59,883 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:59,969 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:00,980 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:01,884 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:02,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:02,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:02,979 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:03,978 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:04,974 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:05,682 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:44:05,986 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:06,105 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:06,106 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:07,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:07,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:08,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:09,001 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:09,887 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:10,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:11,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:11,888 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:12,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:13,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:13,240 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:14,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:15,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:16,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:17,004 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:18,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:19,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:19,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:20,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:21,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:21,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:21,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:22,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:23,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:24,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:24,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:25,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:26,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:27,010 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:28,009 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:29,010 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:29,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:30,011 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:31,011 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:32,013 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:33,013 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:34,014 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:35,017 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:35,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:35,682 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:44:36,015 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:36,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:36,067 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:37,046 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:38,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:38,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:44:38,089 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:44:38,090 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 with policy now +2025-01-18 19:44:38,090 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:44:38,090 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:44:38,090 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:44:38,348 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/bhfc5gf8-media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:38,899 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:44:39,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:39,192 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/svdgsjlo-media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:39,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:40,053 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:40,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:41,048 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:41,222 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:42,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:43,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:43,902 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:44,112 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:45,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:46,079 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:46,257 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:46,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:44:47,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:47,172 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:44:47,172 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:44:47,184 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:44:47,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:44:47,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:48,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:49,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:49,904 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:50,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:51,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:51,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:51,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:52,124 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:52,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:53,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:53,907 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:54,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:55,138 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:56,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:57,170 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:57,265 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:58,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:59,183 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:00,191 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:01,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:02,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:02,939 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:03,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:03,911 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:04,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:05,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:05,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:45:06,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:06,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:06,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:07,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:08,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:08,246 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:09,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:10,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:11,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:11,915 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:12,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:13,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:13,317 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:13,916 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:14,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:15,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:16,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:17,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:17,918 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:18,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:18,471 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:19,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:19,919 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:20,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:21,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:21,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:21,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:21,920 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:22,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:23,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:24,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:24,246 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:25,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:26,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:27,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:28,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:29,178 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:29,598 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:29,923 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:30,033 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:45:30,033 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 with policy now +2025-01-18 19:45:30,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:45:30,041 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:45:30,041 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:45:30,041 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:45:30,181 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:30,295 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/mhbibk0n-media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:45:31,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:31,224 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/hx5ju56f-media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:31,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:32,205 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:33,264 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:34,241 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:35,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:35,593 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:35,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:45:35,926 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:36,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:36,094 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:36,268 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:37,230 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:38,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:39,279 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:39,928 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:40,298 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:40,974 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:41,318 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:42,310 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:43,295 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:44,240 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:45:44,241 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:45:44,241 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:45:44,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:44,930 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:45:44,930 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:45:45,176 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/vwx6wx3m-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:45:45,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:45,931 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:46,261 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:46,333 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:47,335 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:47,931 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:48,340 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:49,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:50,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:51,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:51,093 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:51,295 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:51,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:51,933 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:52,369 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:53,394 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:54,380 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:55,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:56,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:57,292 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:57,397 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:58,432 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:59,452 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:00,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:01,400 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:02,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:03,098 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:03,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:03,938 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:04,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:05,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:05,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:46:06,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:06,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:06,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:07,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:08,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:08,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:09,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:10,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:11,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:12,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:13,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:14,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:14,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:15,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:16,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:17,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:18,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:19,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:19,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:20,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:21,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:21,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:21,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:22,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:23,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:24,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:24,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:25,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:26,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:27,406 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:28,406 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:29,426 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:29,671 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:30,117 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:46:30,117 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 with policy now +2025-01-18 19:46:30,127 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:46:30,128 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:46:30,128 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:46:30,128 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:46:30,378 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/slp670pq-media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:30,454 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:46:31,251 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/8pwr0wse-media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:31,457 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:31,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:32,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:33,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:34,469 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:34,861 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:35,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:35,684 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:46:35,952 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:36,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:36,091 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:36,493 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:37,470 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:38,516 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:39,482 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:39,954 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:40,293 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:40,502 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:41,560 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:42,308 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:46:42,308 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:46:42,309 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:46:42,510 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:42,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:46:42,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:46:43,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:43,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:44,536 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:45,540 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:45,717 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:45,956 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:46,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:47,533 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:48,565 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:49,547 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:49,958 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:50,569 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:51,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:51,103 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:51,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:51,561 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:52,564 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:53,581 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:54,596 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:55,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:56,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:56,600 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:57,597 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:58,649 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:59,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:00,626 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:01,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:01,963 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:02,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:02,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:03,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:03,964 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:04,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:05,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:05,684 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:47:06,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:06,078 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:06,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:07,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:07,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:08,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:09,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:10,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:11,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:12,285 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:12,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:13,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:14,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:15,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:16,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:17,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:18,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:18,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:19,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:20,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:21,078 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:21,078 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:21,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:22,610 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:23,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:23,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:23,972 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:24,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:25,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:26,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:27,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:28,657 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:28,957 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:29,146 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:47:29,146 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 with policy now +2025-01-18 19:47:29,176 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:47:29,176 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:47:29,177 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:47:29,177 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:47:29,407 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7wmngbix-media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:29,662 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:47:30,272 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/w9q0pag3-media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:30,663 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:30,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:31,634 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:31,977 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:32,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:33,667 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:34,746 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:34,957 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:35,685 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:47:35,739 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:35,979 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:36,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:36,103 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:36,703 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:37,706 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:37,980 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:38,733 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:39,733 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:40,341 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:40,376 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:47:40,376 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:47:40,377 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:47:40,786 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:40,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:47:40,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:47:41,738 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:41,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:42,746 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:43,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:43,982 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:44,761 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:45,737 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:46,270 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:46,758 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:47,762 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:47,984 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:48,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:49,785 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:50,756 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:51,127 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:51,127 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:51,355 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:51,789 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:52,803 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:53,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:54,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:55,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:55,987 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:56,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:57,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:57,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:57,989 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:58,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:59,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:00,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:01,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:02,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:02,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:03,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:03,991 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:04,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:05,685 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:48:05,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:06,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:06,080 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:06,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:07,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:07,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:08,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:09,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:10,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:11,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:12,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:12,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:13,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:14,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:15,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:16,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:17,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:17,901 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:18,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:19,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:19,999 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:20,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:21,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:21,081 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:21,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:22,796 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:22,906 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:23,352 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:23,353 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 with policy now +2025-01-18 19:48:23,367 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:23,368 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:23,368 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:23,368 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:23,615 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/6kkt3sa8-media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:23,771 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:24,002 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:48:24,278 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zjr2rw51-media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:24,803 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:25,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:25,813 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:26,002 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:26,854 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:27,855 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:27,933 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:28,821 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:29,828 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:30,004 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:30,853 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:31,851 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:32,005 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:32,829 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:33,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:33,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:34,873 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:35,686 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:48:35,886 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:36,086 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:36,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:36,868 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:37,448 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:37,448 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:37,449 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:37,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:38,007 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:38,007 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:38,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:38,913 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:39,269 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:48:39,269 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:48:39,472 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:39,707 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxt9zj5z_ +2025-01-18 19:48:39,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:39,939 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:48:39,981 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:40,008 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:40,008 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table +2025-01-18 19:48:40,081 INFO SenderThread:2561591 [sender.py:send_request_log_artifact():1525] logged artifact run-138tidw7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTQ1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA0Mzg3Nw==', 'latestArtifact': None}} +2025-01-18 19:48:40,081 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:40,081 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json with policy now +2025-01-18 19:48:40,082 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:40,082 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,083 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,083 DEBUG SenderThread:2561591 [sender.py:send():379] send: telemetry +2025-01-18 19:48:40,083 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,083 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,084 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,084 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,084 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,084 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,085 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,086 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,086 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,086 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,086 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,087 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,087 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,088 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,088 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,089 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,089 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:48:40,310 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7h77vpfn-media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:40,317 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/57l1z76y-media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,330 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jd2jbmbt-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:40,514 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3s55t847 +2025-01-18 19:48:40,565 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/lslxpsj3-media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:40,927 INFO SenderThread:2561591 [sender.py:send_request_log_artifact():1525] logged artifact run-138tidw7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTQ2MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA0Mzg4MA==', 'latestArtifact': None}} +2025-01-18 19:48:40,927 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:40,927 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json with policy now +2025-01-18 19:48:40,927 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:40,928 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,928 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:41,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:41,216 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/txdpo6ti-media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:41,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:42,010 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:42,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:43,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:44,359 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:44,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:45,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:46,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:47,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:48,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:49,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:50,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:50,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:51,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:51,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:51,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:52,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:53,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:54,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:55,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:55,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:56,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:57,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:58,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:59,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:00,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:00,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:01,020 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:49:01,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:02,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:03,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:04,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:05,686 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:49:05,687 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:05,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:06,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:49:06,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:49:06,222 DEBUG SenderThread:2561591 [sender.py:send():379] send: exit +2025-01-18 19:49:06,222 INFO SenderThread:2561591 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:send_exit():588] handling runtime: 1110 +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:send_exit():594] send defer +2025-01-18 19:49:06,223 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 19:49:06,224 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 1 +2025-01-18 19:49:06,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 19:49:06,224 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 2 +2025-01-18 19:49:06,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 19:49:06,224 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 19:49:06,225 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 19:49:06,225 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 19:49:06,225 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 19:49:06,275 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 19:49:06,275 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 19:49:06,276 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined network monitor +2025-01-18 19:49:06,276 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,276 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 19:49:06,276 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 3 +2025-01-18 19:49:06,276 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:49:06,276 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,276 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 19:49:06,277 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,277 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 19:49:06,277 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 4 +2025-01-18 19:49:06,277 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,277 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 19:49:07,023 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:08,024 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,467 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 19:49:12,467 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 5 +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:49:12,468 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,468 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 19:49:12,468 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:12,474 DEBUG SenderThread:2561591 [sender.py:send():379] send: summary +2025-01-18 19:49:12,474 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:12,474 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 6 +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,475 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 19:49:12,475 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 7 +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,475 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 19:49:12,475 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,476 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 19:49:13,029 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:13,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 8 +2025-01-18 19:49:14,905 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:14,905 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:14,905 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 19:49:14,905 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 19:49:14,908 INFO SenderThread:2561591 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 19:49:14,910 INFO SenderThread:2561591 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 19:49:14,925 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 9 +2025-01-18 19:49:14,925 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:14,925 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 19:49:14,926 DEBUG SenderThread:2561591 [sender.py:send():379] send: artifact +2025-01-18 19:49:15,030 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:15,156 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:15,377 INFO wandb-upload_2:2561591 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpauwgphec +2025-01-18 19:49:15,379 INFO wandb-upload_0:2561591 [upload_job.py:push():85] Skipped uploading /tmp/tmptxvvuw2w/wandb-job.json +2025-01-18 19:49:15,527 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp47q_99wi +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTkxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODMzOQ==', 'versionIndex': 447}}} +2025-01-18 19:49:16,473 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml config.yaml +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json wandb-summary.json +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt requirements.txt +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json wandb-metadata.json +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log output.log +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch diff.patch +2025-01-18 19:49:17,035 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 10 +2025-01-18 19:49:17,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,040 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 19:49:17,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 19:49:17,041 INFO SenderThread:2561591 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:49:17,156 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,156 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,211 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt +2025-01-18 19:49:17,230 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:17,232 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:17,243 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:49:17,444 INFO Thread-11 (_thread_body):2561591 [sender.py:transition_state():614] send defer: 11 +2025-01-18 19:49:17,444 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,444 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 19:49:17,444 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,444 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 19:49:17,444 INFO SenderThread:2561591 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 19:49:17,448 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 12 +2025-01-18 19:49:17,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,449 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 19:49:17,449 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,449 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 19:49:17,449 INFO SenderThread:2561591 [file_stream.py:finish():601] file stream finish called +2025-01-18 19:49:17,623 INFO SenderThread:2561591 [file_stream.py:finish():605] file stream finish is done +2025-01-18 19:49:17,623 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 13 +2025-01-18 19:49:17,623 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,623 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 14 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send():379] send: final +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send():379] send: footer +2025-01-18 19:49:17,624 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,624 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,625 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,626 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: server_info +2025-01-18 19:49:17,627 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 19:49:17,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 19:49:17,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:17,687 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,687 INFO MainThread:2561591 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 19:49:17,688 INFO MainThread:2561591 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 19:49:17,688 INFO MainThread:2561591 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 19:49:17,688 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 19:49:17,688 INFO HandlerThread:2561591 [handler.py:finish():882] shutting down handler +2025-01-18 19:49:18,626 INFO WriterThread:2561591 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [sender.py:finish():1615] shutting down sender +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [file_pusher.py:join():175] waiting for file pusher diff --git a/138tidw7/wandb/debug.log b/138tidw7/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7f75e1d9af45815900ae1d8a2404cf269a55b6c2 --- /dev/null +++ b/138tidw7/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Configure stats pid to 2561402 +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/settings +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug.log +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug-internal.log +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7'} +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():619] starting backend +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():623] setting up manager +2025-01-18 19:30:35,311 INFO MainThread:2561402 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:30:35,312 INFO MainThread:2561402 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:30:35,324 INFO MainThread:2561402 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:30:35,332 INFO MainThread:2561402 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:30:35,591 INFO MainThread:2561402 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:30:35,656 INFO MainThread:2561402 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:30:35,657 INFO MainThread:2561402 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:30:36,026 INFO MainThread:2561402 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:30:36,027 INFO MainThread:2561402 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:30:36,028 INFO MainThread:2561402 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:30:36,028 INFO MainThread:2561402 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:30:36,029 INFO MainThread:2561402 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:30:36,029 INFO MainThread:2561402 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/tb, True +2025-01-18 19:30:44,487 INFO MainThread:2561402 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 19:49:18,689 WARNING MsgRouterThr:2561402 [router.py:message_loop():77] message_loop has been closed diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py b/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml b/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb6eae7f4b3c7c0a3f799174245cdf25b7dc4ef3 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_0' +wandb_notes: + desc: null + value: 3465_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737246635 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch b/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..2072e33624d053e0f6dcae1f53eb01ea76c3e97c --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch @@ -0,0 +1,93 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..cda0fb8 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,18 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d65485d 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 8000 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9b7f5ad82f4e6e2e31d9f6b746e5ecffb4f2bfe4 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.05, 0.000599756936771223, 0.2553238273156284, 0.0014435235476719104, 0.19057887798775708, 0.01022623675426293, 0.18441923714035235, 0.008670444698550012, 0.17789256951694576]]} \ No newline at end of file diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json new file mode 100644 index 0000000000000000000000000000000000000000..40ff2366b98823c90738bb5366d6a5ddb9e52c07 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[14, 40, 0.35, 0.19708785420753988, 0.5364554226325556, 0.21984018391013505, 0.49576589465225984, 0.2465912141412675, 0.47091083974227943, 0.24901367356432502, 0.45518821851315283]]} \ No newline at end of file diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d5a524fa702400ae0fe8fd95309174bd7aa60db2 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96591af35a54c45302d0a9395a677d03a00e4e3c32409560bff49f0868d21c7a +size 1111368 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f8d9a9f5a939836fc5492a52b8131f907004b49 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b221e9eaaa2afeafab23d9f82015bfca05527b928936906a55e5a9e06446ed +size 1379873 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3eb19ac955aa4562476ecd62267a99db16fb2fb8 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d95850d1df3e2b923407436043bd539b9210471db5dff5ae624216157c3095 +size 1448723 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c97379d5846507462e626be6035d2325093d9247 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11b1f9ef1d691b07f19acb5f6dbd0edb5a56defa33558799db6d69e9167a70f5 +size 1499478 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a439658d866263626da781e8e0c17cc1f1a497f --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a298f612d50a7b95e2210ba8a86026cb6debdb5f035ded03455ff2f7be1b933e +size 1453691 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2336253fd14f6791dbd67f5451255f1e20fe751b --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da508fd72acfd8e8c94ec6cd36d8a75233551e542aa1e99546a31fee0e34f92f +size 1515972 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..204b0d1c73a1d5b669ad3e5ff3af1f425c530208 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f145ad74c481c07c43b6f16cbb657074336b35655bdd0e29efad994b960229 +size 1629682 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4b1d727fbb86c5b1f4f983ed07e87584d4b40f15 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4567bcc512ee0512f3ae490be445b8a9b07cd9ca027fe3726d8f4b23836ee34f +size 1619329 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4fad187a04303e93dbfe6c352b1ac00a8ef9cf5f --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415f21e10d2661428c434b6830ac89b499e81e0e2ac28b80f4367e3d27fcb141 +size 1693625 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a0689c3e8b469193aa63bad424d473a5e694b75 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c84898aed318ecd95cd83b9d459daea4ca54b8963dedad94016c4ace6718e58 +size 1713057 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ca04b3f0ab4459ed29621eadc5e3750a5340813 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d5fa28d764eeaa07d1aec6e42f028f7e27173f4955f7b3bc8d442c5d180f5f +size 1480591 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..76fe0aa1cae95f3ecec3e2ed8cffbc5238e4133a --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0aac6515bc4a872c577de09d43dec9134d3a60ceedfb842c27ae3a77272c2a5 +size 1193043 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b623019bb7287ef8a554c263200329eaf0bc2b94 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcfa4207d555c0dc155d3bcf40a93b4285f85d8425c2686d714270bc309487ee +size 1211592 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9edd55caef748e7bfb0ec0630e51f2550793fb4 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6854d6b9b0ee461bdf8cf13ae97bc2a808f179bc878f37657dba33fd07640c29 +size 1157029 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5d7818f6554484e0684ea974d4abca11a3b2519 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573f9d77908239cd99f44510bb179b8515f548ec362d12ceccdd2a47e32d1188 +size 1220051 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb81252d15ee87c5b254657560e04bc249a8d8bf --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6b889b8499cd4be78a68284395f6b7d7bcab408235e139ea67590b4af6cd73 +size 1123543 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..079d7ef73fa777c605e9d2cce8cd7ae91c61b867 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d85fdaf1b462a9c0774327705488b9c5ab10fc274b3c27ec53c7f18a4d56263 +size 1199797 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af0e02cf7d327f8895cdef8a0790c40a1c8b399d --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bd94bd9f2e25c7a138f737e6e83ed926b3c8a185481378cdfdded93f5e3dfe +size 1296513 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e0370b78029d97882755908942cc6cd7320854a --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad80c418d3b655ed57abd05bd86565ae377f9fb98ccfaee5a538d56a4bd2253 +size 1348396 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65f1cf8b62d35d89c1f5faafce297f7a1a2920a6 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cc9e9466280c44c4ea9fe175a9c581ee927269612a62b0329244346f94b32e6 +size 1234297 diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log b/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..98c312b84f4b0b000ae4117d842d0f36bb284ddd --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log @@ -0,0 +1,1034 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737246644.487964]: resetting environment +[INFO] [1737246644.492879]: cleared current plan +[INFO] [1737246644.494119]: moving to home +[INFO] [1737246645.496794]: reached home +[INFO] [1737246645.498026]: reset action history +[INFO] [1737246648.206750]: environment reset +Starting episode 0 +[INFO] [1737246648.208734]: resetting environment +[INFO] [1737246648.214001]: cleared current plan +[INFO] [1737246648.215583]: moving to home +[INFO] [1737246649.217295]: reached home +[INFO] [1737246649.219062]: reset action history +[INFO] [1737246651.928353]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1737246678.029204]: resetting environment +[INFO] [1737246678.035316]: cleared current plan +[INFO] [1737246678.036189]: moving to home +[INFO] [1737246679.038380]: reached home +[INFO] [1737246679.040829]: reset action history +[INFO] [1737246681.798880]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_0_2025-01-18-19-31-21.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:43057/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2561709] +process[print_text-2]: started with pid [2561732] +process[rosbag_record-3]: started with pid [2561733] +started bagging! +[INFO] [1737246708.794198]: Storing episode... +[rosbag_record-3] killing on exit[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737246709.517461]: Stored episode 1. +[INFO] [1737246709.521519]: Saving video... +[INFO] [1737246709.718297]: Video saved! +Starting episode 1 +[INFO] [1737246709.735210]: resetting environment +[INFO] [1737246709.742178]: cleared current plan +[INFO] [1737246709.742627]: moving to home +[INFO] [1737246712.845408]: reached home +[INFO] [1737246712.845877]: reset action history +[INFO] [1737246715.553794]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246721.427033]: resetting environment +[INFO] [1737246721.432248]: cleared current plan +[INFO] [1737246721.432531]: moving to home +[INFO] [1737246722.433359]: reached home +[INFO] [1737246722.433793]: reset action history +[INFO] [1737246725.142104]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_1_2025-01-18-19-32-05.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:34153/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2561865] +process[print_text-5]: started with pid [2561866] +process[rosbag_record-6]: started with pid [2561867] +started bagging! +For topic gripper_width: timestamp difference is -191543265 for nearest: 1737246740302286701 - target: 1737246740493829966 at idx 193 +For topic gripper_width: timestamp difference is -108330679 for nearest: 1737246748402285623 - target: 1737246748510616302 at idx 249 +For topic gripper_width: timestamp difference is -110726411 for nearest: 1737246752202285474 - target: 1737246752313011885 at idx 249 +For topic gripper_width: timestamp difference is -110529047 for nearest: 1737246752402320045 - target: 1737246752512849092 at idx 249 +[INFO] [1737246753.172395]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737246753.968461]: Stored episode 2. +[INFO] [1737246753.970526]: Saving video... +[INFO] [1737246754.171782]: Video saved! +Starting episode 2 +[INFO] [1737246754.208619]: resetting environment +[INFO] [1737246754.229398]: cleared current plan +[INFO] [1737246754.231230]: moving to home +[INFO] [1737246757.736521]: reached home +[INFO] [1737246757.740943]: reset action history +[INFO] [1737246760.518873]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246771.666187]: resetting environment +[INFO] [1737246771.678574]: cleared current plan +[INFO] [1737246771.679678]: moving to home +[INFO] [1737246772.683076]: reached home +[INFO] [1737246772.686867]: reset action history +[INFO] [1737246775.462817]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_2_2025-01-18-19-32-55.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:34539/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2562005] +process[print_text-8]: started with pid [2562006] +process[rosbag_record-9]: started with pid [2562007] +started bagging! +For topic gripper_width: timestamp difference is -104546273 for nearest: 1737246801002283847 - target: 1737246801106830120 at idx 249 +[INFO] [1737246803.912138]: Storing episode... +[rosbag_record-9] killing on exit[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1737246804.685419]: Stored episode 3. +[INFO] [1737246804.688349]: Saving video... +[INFO] [1737246804.912839]: Video saved! +Starting episode 3 +[INFO] [1737246804.947582]: resetting environment +[INFO] [1737246804.968762]: cleared current plan +[INFO] [1737246804.972640]: moving to home +[INFO] [1737246808.285299]: reached home +[INFO] [1737246808.289404]: reset action history +[INFO] [1737246811.057205]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246817.512591]: resetting environment +[INFO] [1737246817.528879]: cleared current plan +[INFO] [1737246817.546834]: moving to home +[INFO] [1737246818.555977]: reached home +[INFO] [1737246818.558680]: reset action history +[INFO] [1737246821.280688]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_3_2025-01-18-19-33-41.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:33835/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2562138] +process[print_text-11]: started with pid [2562139] +process[rosbag_record-12]: started with pid [2562140] +started bagging! +[INFO] [1737246849.663455]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737246850.399433]: Stored episode 4. +[INFO] [1737246850.408310]: Saving video... +[INFO] [1737246850.601022]: Video saved! +Starting episode 4 +[INFO] [1737246850.658426]: resetting environment +[INFO] [1737246850.674310]: cleared current plan +[INFO] [1737246850.678561]: moving to home +[INFO] [1737246854.101351]: reached home +[INFO] [1737246854.117144]: reset action history +[INFO] [1737246856.947592]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246865.430809]: resetting environment +[INFO] [1737246865.450469]: cleared current plan +[INFO] [1737246865.452754]: moving to home +[INFO] [1737246866.455698]: reached home +[INFO] [1737246866.457244]: reset action history +[INFO] [1737246869.250835]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_4_2025-01-18-19-34-29.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:40269/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2562276] +process[print_text-14]: started with pid [2562277] +process[rosbag_record-15]: started with pid [2562278] +started bagging! +[INFO] [1737246897.316031]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1737246898.092498]: Stored episode 5. +[INFO] [1737246898.111692]: Saving video... +[INFO] [1737246898.340993]: Video saved! +Starting episode 5 +[INFO] [1737246898.389872]: resetting environment +[INFO] [1737246898.425331]: cleared current plan +[INFO] [1737246898.426279]: moving to home +[INFO] [1737246901.634620]: reached home +[INFO] [1737246901.637015]: reset action history +[INFO] [1737246904.433758]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246906.526546]: resetting environment +[INFO] [1737246906.566211]: cleared current plan +[INFO] [1737246906.573246]: moving to home +[INFO] [1737246907.579365]: reached home +[INFO] [1737246907.601388]: reset action history +[INFO] [1737246910.457676]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_5_2025-01-18-19-35-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:37537/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2562409] +process[print_text-17]: started with pid [2562410] +process[rosbag_record-18]: started with pid [2562411] +started bagging! +For topic gripper_width: timestamp difference is -237635065 for nearest: 1737246925202285837 - target: 1737246925439920902 at idx 205 +For topic gripper_width: timestamp difference is -232846151 for nearest: 1737246931002372136 - target: 1737246931235218287 at idx 249 +[INFO] [1737246938.109524]: Storing episode... +[rosbag_record-18] killing on exit +[pub_text-16] killing on exit +[print_text-17] killing on exit +[INFO] [1737246938.878720]: Stored episode 6. +[INFO] [1737246938.886224]: Saving video... +[INFO] [1737246939.116384]: Video saved! +Starting episode 6 +[INFO] [1737246939.190528]: resetting environment +[INFO] [1737246939.211398]: cleared current plan +[INFO] [1737246939.212875]: moving to home +[INFO] [1737246942.619894]: reached home +[INFO] [1737246942.626473]: reset action history +[INFO] [1737246945.546708]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737246960.793088]: resetting environment +[INFO] [1737246960.829431]: cleared current plan +[INFO] [1737246960.843341]: moving to home +[INFO] [1737246961.852201]: reached home +[INFO] [1737246961.862575]: reset action history +[INFO] [1737246964.731469]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_6_2025-01-18-19-36-04.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:44859/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2562543] +process[print_text-20]: started with pid [2562544] +process[rosbag_record-21]: started with pid [2562545] +started bagging! +For topic gripper_width: timestamp difference is -227519881 for nearest: 1737246979002284873 - target: 1737246979229804754 at idx 204 +For topic gripper_width: timestamp difference is -124881551 for nearest: 1737246982302279427 - target: 1737246982427160978 at idx 245 +For topic gripper_width: timestamp difference is -222960489 for nearest: 1737246984602304680 - target: 1737246984825265169 at idx 249 +[INFO] [1737246992.901639]: Storing episode... +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[pub_text-19] killing on exit +[INFO] [1737246993.675978]: Stored episode 7. +[INFO] [1737246993.699967]: Saving video... +[INFO] [1737246993.927819]: Video saved! +Starting episode 7 +[INFO] [1737246994.003402]: resetting environment +[INFO] [1737246994.020774]: cleared current plan +[INFO] [1737246994.021028]: moving to home +[INFO] [1737246997.233922]: reached home +[INFO] [1737246997.235248]: reset action history +[INFO] [1737247000.131030]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247000.546617]: resetting environment +[INFO] [1737247000.563477]: cleared current plan +[INFO] [1737247000.567033]: moving to home +[INFO] [1737247001.574180]: reached home +[INFO] [1737247001.588392]: reset action history +[INFO] [1737247004.498875]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_7_2025-01-18-19-36-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:35419/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2562677] +process[print_text-23]: started with pid [2562678] +process[rosbag_record-24]: started with pid [2562679] +started bagging! +For topic gripper_width: timestamp difference is 102842910 for nearest: 1737247012569044215 - target: 1737247012466201305 at idx 122 +For topic gripper_width: timestamp difference is 103200551 for nearest: 1737247012768974181 - target: 1737247012665773630 at idx 124 +For topic gripper_width: timestamp difference is 104394346 for nearest: 1737247013368965536 - target: 1737247013264571190 at idx 132 +[INFO] [1737247036.569820]: Storing episode... +[print_text-23] killing on exit +[rosbag_record-24] killing on exit +[pub_text-22] killing on exit +[INFO] [1737247037.408725]: Stored episode 8. +[INFO] [1737247037.415577]: Saving video... +[INFO] [1737247037.635618]: Video saved! +Starting episode 8 +[INFO] [1737247037.711846]: resetting environment +[INFO] [1737247037.777887]: cleared current plan +[INFO] [1737247037.800475]: moving to home +[INFO] [1737247041.214141]: reached home +[INFO] [1737247041.218477]: reset action history +[INFO] [1737247044.201491]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247046.225320]: resetting environment +[INFO] [1737247046.284128]: cleared current plan +[INFO] [1737247046.292441]: moving to home +[INFO] [1737247047.300260]: reached home +[INFO] [1737247047.331131]: reset action history +[INFO] [1737247050.267963]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_8_2025-01-18-19-37-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:38201/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2562809] +process[print_text-26]: started with pid [2562810] +process[rosbag_record-27]: started with pid [2562811] +started bagging! +For topic gripper_width: timestamp difference is -108537712 for nearest: 1737247057702305994 - target: 1737247057810843706 at idx 100 +For topic gripper_width: timestamp difference is -100513301 for nearest: 1737247062302306332 - target: 1737247062402819633 at idx 162 +[INFO] [1737247083.464479]: Storing episode... +[print_text-26] killing on exit +[rosbag_record-27] killing on exit +[pub_text-25] killing on exit +[INFO] [1737247084.237894]: Stored episode 9. +[INFO] [1737247084.264078]: Saving video... +[INFO] [1737247084.460023]: Video saved! +Starting episode 9 +[INFO] [1737247084.560555]: resetting environment +[INFO] [1737247084.581540]: cleared current plan +[INFO] [1737247084.585398]: moving to home +[INFO] [1737247088.101294]: reached home +[INFO] [1737247088.104773]: reset action history +[INFO] [1737247091.060939]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247108.220257]: resetting environment +[INFO] [1737247108.243248]: cleared current plan +[INFO] [1737247108.246996]: moving to home +[INFO] [1737247109.253396]: reached home +[INFO] [1737247109.268338]: reset action history +[INFO] [1737247112.244083]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_9_2025-01-18-19-38-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:35903/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2562943] +process[print_text-29]: started with pid [2562944] +process[rosbag_record-30]: started with pid [2562945] +started bagging! +For topic gripper_width: timestamp difference is 101400381 for nearest: 1737247138468969589 - target: 1737247138367569208 at idx 249 +[INFO] [1737247140.442746]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit +[pub_text-28] killing on exit +[INFO] [1737247141.333810]: Stored episode 10. +[INFO] [1737247141.337570]: Saving video... +[INFO] [1737247141.559268]: Video saved! +Starting episode 10 +[INFO] [1737247141.653121]: resetting environment +[INFO] [1737247141.691581]: cleared current plan +[INFO] [1737247141.704683]: moving to home +[INFO] [1737247145.017310]: reached home +[INFO] [1737247145.048153]: reset action history +[INFO] [1737247148.175769]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247151.970828]: resetting environment +[INFO] [1737247152.010473]: cleared current plan +[INFO] [1737247152.012663]: moving to home +[INFO] [1737247153.035289]: reached home +[INFO] [1737247153.049417]: reset action history +[INFO] [1737247156.013160]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_10_2025-01-18-19-39-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:34705/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2563078] +process[print_text-32]: started with pid [2563079] +process[rosbag_record-33]: started with pid [2563080] +started bagging! +For topic gripper_width: timestamp difference is 102462996 for nearest: 1737247178005101193 - target: 1737247177902638197 at idx 249 +For topic gripper_width: timestamp difference is 102575746 for nearest: 1737247181602278438 - target: 1737247181499702692 at idx 249 +[INFO] [1737247188.596199]: Storing episode... +[print_text-32] killing on exit +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[INFO] [1737247189.376809]: Stored episode 11. +[INFO] [1737247189.390944]: Saving video... +[INFO] [1737247189.618297]: Video saved! +Starting episode 11 +[INFO] [1737247189.762056]: resetting environment +[INFO] [1737247189.806695]: cleared current plan +[INFO] [1737247189.810669]: moving to home +[INFO] [1737247193.317258]: reached home +[INFO] [1737247193.345908]: reset action history +[INFO] [1737247196.424557]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247208.330486]: resetting environment +[INFO] [1737247208.419891]: cleared current plan +[INFO] [1737247208.429396]: moving to home +[INFO] [1737247209.452079]: reached home +[INFO] [1737247209.469380]: reset action history +[INFO] [1737247212.567001]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_11_2025-01-18-19-40-12.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:46275/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2563222] +process[print_text-35]: started with pid [2563223] +process[rosbag_record-36]: started with pid [2563224] +started bagging! +[INFO] [1737247247.066526]: Storing episode... +[rosbag_record-36] killing on exit +[print_text-35] killing on exit[pub_text-34] killing on exit +[INFO] [1737247247.915295]: Stored episode 12. +[INFO] [1737247247.934679]: Saving video... +[INFO] [1737247248.189625]: Video saved! +[WARN] [1737247248.435545]: Plan exhausted +Starting episode 12 +[INFO] [1737247248.378043]: resetting environment +[INFO] [1737247248.444092]: cleared current plan +[INFO] [1737247248.454895]: moving to home +[INFO] [1737247251.978634]: reached home +[INFO] [1737247252.009116]: reset action history +[INFO] [1737247255.112365]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247262.092111]: resetting environment +[INFO] [1737247262.167306]: cleared current plan +[INFO] [1737247262.199222]: moving to home +[INFO] [1737247263.225191]: reached home +[INFO] [1737247263.252373]: reset action history +[INFO] [1737247266.424981]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_12_2025-01-18-19-41-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:40389/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2563368] +process[print_text-38]: started with pid [2563369] +process[rosbag_record-39]: started with pid [2563370] +started bagging! +[INFO] [1737247301.622645]: Storing episode... +[rosbag_record-39] killing on exit +[pub_text-37] killing on exit +[print_text-38] killing on exit +[INFO] [1737247302.483948]: Stored episode 13. +[INFO] [1737247302.520264]: Saving video... +[INFO] [1737247302.729909]: Video saved! +Starting episode 13 +[INFO] [1737247302.894897]: resetting environment +[INFO] [1737247302.931461]: cleared current plan +[INFO] [1737247302.968819]: moving to home +[INFO] [1737247306.499208]: reached home +[INFO] [1737247306.503288]: reset action history +[INFO] [1737247309.619354]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247322.136192]: resetting environment +[INFO] [1737247322.215209]: cleared current plan +[INFO] [1737247322.244797]: moving to home +[INFO] [1737247323.269933]: reached home +[INFO] [1737247323.284342]: reset action history +[INFO] [1737247326.542167]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_13_2025-01-18-19-42-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:36409/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2563515] +process[print_text-41]: started with pid [2563516] +process[rosbag_record-42]: started with pid [2563517] +started bagging! +For topic gripper_width: timestamp difference is -102899781 for nearest: 1737247339702276716 - target: 1737247339805176497 at idx 201 +For topic gripper_width: timestamp difference is -101614906 for nearest: 1737247341302315520 - target: 1737247341403930426 at idx 224 +For topic gripper_width: timestamp difference is -106611641 for nearest: 1737247349702304213 - target: 1737247349808915854 at idx 249 +[INFO] [1737247361.663662]: Storing episode... +[rosbag_record-42] killing on exit +[print_text-41] killing on exit +[pub_text-40] killing on exit +[WARN] [1737247363.002452]: Plan exhausted +[WARN] [1737247363.036382]: Plan exhausted +[INFO] [1737247362.608214]: Stored episode 14. +[INFO] [1737247362.616823]: Saving video... +[INFO] [1737247362.851514]: Video saved! +Starting episode 14 +[INFO] [1737247363.009858]: resetting environment +[INFO] [1737247363.044869]: cleared current plan +[INFO] [1737247363.046839]: moving to home +[INFO] [1737247366.467179]: reached home +[INFO] [1737247366.474317]: reset action history +[INFO] [1737247369.669852]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247378.452416]: resetting environment +[INFO] [1737247378.470832]: cleared current plan +[INFO] [1737247378.471592]: moving to home +[INFO] [1737247379.482770]: reached home +[INFO] [1737247379.495911]: reset action history +[INFO] [1737247382.731041]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_14_2025-01-18-19-43-02.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:40675/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2563660] +process[print_text-44]: started with pid [2563661] +process[rosbag_record-45]: started with pid [2563662] +started bagging! +For topic gripper_width: timestamp difference is -123938445 for nearest: 1737247396102269288 - target: 1737247396226207733 at idx 189 +[INFO] [1737247420.269035]: Storing episode... +[rosbag_record-45] killing on exit +[pub_text-43] killing on exit +[print_text-44] killing on exit +[WARN] [1737247421.435241]: Plan exhausted +[WARN] [1737247421.474926]: Plan exhausted +[WARN] [1737247421.517432]: Plan exhausted +[WARN] [1737247421.572064]: Plan exhausted +[WARN] [1737247421.601709]: Plan exhausted +[INFO] [1737247421.111159]: Stored episode 15. +[INFO] [1737247421.112961]: Saving video... +[INFO] [1737247421.344956]: Video saved! +Starting episode 15 +[INFO] [1737247421.566510]: resetting environment +[INFO] [1737247421.606574]: cleared current plan +[INFO] [1737247421.624182]: moving to home +[INFO] [1737247425.139957]: reached home +[INFO] [1737247425.173805]: reset action history +[INFO] [1737247428.401364]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247434.176341]: resetting environment +[INFO] [1737247434.249893]: cleared current plan +[INFO] [1737247434.283944]: moving to home +[INFO] [1737247435.319274]: reached home +[INFO] [1737247435.352845]: reset action history +[INFO] [1737247438.553448]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_15_2025-01-18-19-43-58.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:46433/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2563805] +process[print_text-47]: started with pid [2563806] +process[rosbag_record-48]: started with pid [2563807] +started bagging! +[WARN] [1737247477.837458]: Plan exhausted +[WARN] [1737247477.915470]: Plan exhausted +[WARN] [1737247477.925973]: Plan exhausted +[WARN] [1737247477.959633]: Plan exhausted +[WARN] [1737247477.994929]: Plan exhausted +[WARN] [1737247478.043677]: Plan exhausted +[WARN] [1737247478.074928]: Plan exhausted +[WARN] [1737247478.117878]: Plan exhausted +[INFO] [1737247476.682261]: Storing episode... +[print_text-47] killing on exit +[rosbag_record-48] killing on exit +[pub_text-46] killing on exit +[INFO] [1737247477.580648]: Stored episode 16. +[INFO] [1737247477.620116]: Saving video... +[INFO] [1737247477.876521]: Video saved! +Starting episode 16 +[INFO] [1737247478.102138]: resetting environment +[INFO] [1737247478.206212]: cleared current plan +[INFO] [1737247478.219088]: moving to home +[WARN] [1737247478.155652]: Plan exhausted +[INFO] [1737247481.876297]: reached home +[INFO] [1737247481.902767]: reset action history +[INFO] [1737247485.131716]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247486.524058]: resetting environment +[INFO] [1737247486.580707]: cleared current plan +[INFO] [1737247486.580976]: moving to home +[INFO] [1737247487.583352]: reached home +[INFO] [1737247487.614858]: reset action history +[INFO] [1737247490.919280]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_16_2025-01-18-19-44-51.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:41345/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2563949] +process[print_text-50]: started with pid [2563950] +process[rosbag_record-51]: started with pid [2563951] +started bagging! +For topic gripper_width: timestamp difference is 100698253 for nearest: 1737247509668913385 - target: 1737247509568215132 at idx 249 +For topic gripper_width: timestamp difference is 100888359 for nearest: 1737247509868942367 - target: 1737247509768054008 at idx 249 +For topic gripper_width: timestamp difference is 101396038 for nearest: 1737247510468966438 - target: 1737247510367570400 at idx 249 +For topic gripper_width: timestamp difference is 101966699 for nearest: 1737247511268896898 - target: 1737247511166930199 at idx 249 +For topic gripper_width: timestamp difference is 102843386 for nearest: 1737247512268940073 - target: 1737247512166096687 at idx 249 +For topic gripper_width: timestamp difference is 105149281 for nearest: 1737247515068943036 - target: 1737247514963793755 at idx 249 +For topic gripper_width: timestamp difference is 106288490 for nearest: 1737247516468946514 - target: 1737247516362658024 at idx 249 +For topic gripper_width: timestamp difference is 107931383 for nearest: 1737247518468939550 - target: 1737247518361008167 at idx 249 +For topic gripper_width: timestamp difference is 108096328 for nearest: 1737247518668941226 - target: 1737247518560844898 at idx 249 +[INFO] [1737247528.640002]: Storing episode... +[WARN] [1737247529.798007]: Plan exhausted +[WARN] [1737247529.834948]: Plan exhausted +[WARN] [1737247529.880007]: Plan exhausted +[WARN] [1737247529.915201]: Plan exhausted +[WARN] [1737247529.964754]: Plan exhausted +[WARN] [1737247529.999105]: Plan exhausted +[WARN] [1737247530.038829]: Plan exhausted +[WARN] [1737247530.077539]: Plan exhausted +[WARN] [1737247530.115076]: Plan exhausted +[rosbag_record-51] killing on exit +[print_text-50] killing on exit[pub_text-49] killing on exit +[INFO] [1737247529.565990]: Stored episode 17. +[INFO] [1737247529.625894]: Saving video... +[INFO] [1737247529.865541]: Video saved! +Starting episode 17 +[INFO] [1737247530.063847]: resetting environment +[INFO] [1737247530.120992]: cleared current plan +[INFO] [1737247530.124981]: moving to home +[INFO] [1737247533.538622]: reached home +[INFO] [1737247533.568986]: reset action history +[INFO] [1737247536.926787]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247544.336089]: resetting environment +[INFO] [1737247544.391142]: cleared current plan +[INFO] [1737247544.407451]: moving to home +[INFO] [1737247545.424650]: reached home +[INFO] [1737247545.443823]: reset action history +[INFO] [1737247548.878728]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_17_2025-01-18-19-45-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:43173/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2564095] +process[print_text-53]: started with pid [2564096] +process[rosbag_record-54]: started with pid [2564097] +started bagging! +[WARN] [1737247590.164304]: Plan exhausted +[WARN] [1737247590.201659]: Plan exhausted +[INFO] [1737247588.800088]: Storing episode... +[pub_text-52] killing on exit +[rosbag_record-54] killing on exit +[print_text-53] killing on exit +[INFO] [1737247589.666643]: Stored episode 18. +[INFO] [1737247589.679119]: Saving video... +[INFO] [1737247589.927413]: Video saved! +Starting episode 18 +[INFO] [1737247590.135677]: resetting environment +[INFO] [1737247590.223792]: cleared current plan +[INFO] [1737247590.247919]: moving to home +[INFO] [1737247593.811124]: reached home +[INFO] [1737247593.818210]: reset action history +[INFO] [1737247597.241996]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247602.501336]: resetting environment +[INFO] [1737247602.615903]: cleared current plan +[INFO] [1737247602.641640]: moving to home +[INFO] [1737247603.664171]: reached home +[INFO] [1737247603.676990]: reset action history +[INFO] [1737247607.108266]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_18_2025-01-18-19-46-47.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:37197/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2564241] +process[print_text-56]: started with pid [2564242] +process[rosbag_record-57]: started with pid [2564243] +started bagging! +For topic gripper_width: timestamp difference is 107615373 for nearest: 1737247641868937633 - target: 1737247641761322260 at idx 249 +[INFO] [1737247647.657799]: Storing episode... +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit[print_text-56] killing on exit +[INFO] [1737247648.574909]: Stored episode 19. +[INFO] [1737247648.605269]: Saving video... +[WARN] [1737247648.994938]: Plan exhausted +[WARN] [1737247649.036482]: Plan exhausted +[WARN] [1737247649.079742]: Plan exhausted +[WARN] [1737247649.114923]: Plan exhausted +[WARN] [1737247649.167227]: Plan exhausted +[WARN] [1737247649.208251]: Plan exhausted +[WARN] [1737247649.263479]: Plan exhausted +[WARN] [1737247649.299898]: Plan exhausted +[INFO] [1737247648.882992]: Video saved! +Starting episode 19 +[INFO] [1737247649.246886]: resetting environment +[INFO] [1737247649.322619]: cleared current plan +[INFO] [1737247649.337503]: moving to home +[INFO] [1737247652.887759]: reached home +[INFO] [1737247652.925050]: reset action history +[INFO] [1737247656.470283]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737247660.559419]: resetting environment +[INFO] [1737247660.731583]: cleared current plan +[INFO] [1737247660.758527]: moving to home +[INFO] [1737247661.802198]: reached home +[INFO] [1737247661.832408]: reset action history +[INFO] [1737247665.193985]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/episode_rosbags/episode_19_2025-01-18-19-47-45.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2561402.log +started roslaunch server http://158.130.50.37:46647/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2564387] +process[print_text-59]: started with pid [2564388] +process[rosbag_record-60]: started with pid [2564389] +started bagging! +For topic gripper_width: timestamp difference is 103093441 for nearest: 1737247682668890532 - target: 1737247682565797091 at idx 249 +For topic gripper_width: timestamp difference is -247996999 for nearest: 1737247696502296017 - target: 1737247696750293016 at idx 249 +[INFO] [1737247702.032004]: Storing episode... +[print_text-59] killing on exit +[rosbag_record-60] killing on exit +[pub_text-58] killing on exit +[WARN] [1737247703.399055]: Plan exhausted +[INFO] [1737247702.899477]: Stored episode 20. +[INFO] [1737247702.923238]: Saving video... +[INFO] [1737247703.181017]: Video saved! +[INFO] [1737247703.368959]: resetting environment +[INFO] [1737247703.426895]: cleared current plan +[INFO] [1737247703.431782]: moving to home +[INFO] [1737247706.846899]: reached home +[INFO] [1737247706.903313]: reset action history +[INFO] [1737247710.098264]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt b/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..1dbac5b60f6f56f0d85465fa2fc52de7a02ca968 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,110 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..cda0fb8 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,25 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++# wandb_run_id: '3456_2' # seed 2 ++ ++# all books, 06/20 demos per book ++# crp_D+act history ++wandb_run_id: '3465_0' ++# wandb_run_id: '3465_1' # seed 1 ++# wandb_run_id: '3465_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..d65485d 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,15 +167,21 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ # '3456_2', ++ '3465_0', ++ '3465_1', ++ '3465_2', + ] + +-checkpoint_epoch = 12000 ++checkpoint_epoch = 8000 + + # use subprocess to download the model checkpoints in parallel + process_list = [] +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json b/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..96fcae0d5225664492255f3ca949167b388e8e5e --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-19T00:30:35.680913", + "startedAt": "2025-01-19T00:30:35.299465", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3565.9246249999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4520.276, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4515.837, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 550.1006927490234 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json b/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b2718d47b0aaee9888b19fa5424278d6f574fcd3 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "96591af35a54c45302d0a9395a677d03a00e4e3c32409560bff49f0868d21c7a", "size": 1111368, "path": "media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4"}, "global_step": 2656, "_timestamp": 1737247719.8111885, "_runtime": 1084.4981834888458, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "b0aac6515bc4a872c577de09d43dec9134d3a60ceedfb842c27ae3a77272c2a5", "size": 1193043, "path": "media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4"}, "eval/num_success": 1.0, "episode": 40.0, "eval/success_rate": 0.05000000074505806, "eval/success_rate_lb_0.01": 0.0005997569533064961, "eval/success_rate_ub_0.01": 0.2553238272666931, "eval/success_rate_lb_0.025": 0.001443523564375937, "eval/success_rate_ub_0.025": 0.1905788779258728, "eval/success_rate_lb_0.05": 0.010226236656308174, "eval/success_rate_ub_0.05": 0.18441924452781677, "eval/success_rate_lb_0.1": 0.0086704446002841, "eval/success_rate_ub_0.1": 0.1778925657272339, "eval/2_eval": {"_type": "video-file", "sha256": "dcfa4207d555c0dc155d3bcf40a93b4285f85d8425c2686d714270bc309487ee", "size": 1211592, "path": "media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "6854d6b9b0ee461bdf8cf13ae97bc2a808f179bc878f37657dba33fd07640c29", "size": 1157029, "path": "media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "573f9d77908239cd99f44510bb179b8515f548ec362d12ceccdd2a47e32d1188", "size": 1220051, "path": "media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "8f6b889b8499cd4be78a68284395f6b7d7bcab408235e139ea67590b4af6cd73", "size": 1123543, "path": "media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "8d85fdaf1b462a9c0774327705488b9c5ab10fc274b3c27ec53c7f18a4d56263", "size": 1199797, "path": "media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "21bd94bd9f2e25c7a138f737e6e83ed926b3c8a185481378cdfdded93f5e3dfe", "size": 1296513, "path": "media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "8ad80c418d3b655ed57abd05bd86565ae377f9fb98ccfaee5a538d56a4bd2253", "size": 1348396, "path": "media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "3cc9e9466280c44c4ea9fe175a9c581ee927269612a62b0329244346f94b32e6", "size": 1234297, "path": "media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "b5b221e9eaaa2afeafab23d9f82015bfca05527b928936906a55e5a9e06446ed", "size": 1379873, "path": "media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "60d95850d1df3e2b923407436043bd539b9210471db5dff5ae624216157c3095", "size": 1448723, "path": "media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "11b1f9ef1d691b07f19acb5f6dbd0edb5a56defa33558799db6d69e9167a70f5", "size": 1499478, "path": "media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "a298f612d50a7b95e2210ba8a86026cb6debdb5f035ded03455ff2f7be1b933e", "size": 1453691, "path": "media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "da508fd72acfd8e8c94ec6cd36d8a75233551e542aa1e99546a31fee0e34f92f", "size": 1515972, "path": "media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "f8f145ad74c481c07c43b6f16cbb657074336b35655bdd0e29efad994b960229", "size": 1629682, "path": "media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "4567bcc512ee0512f3ae490be445b8a9b07cd9ca027fe3726d8f4b23836ee34f", "size": 1619329, "path": "media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "415f21e10d2661428c434b6830ac89b499e81e0e2ac28b80f4367e3d27fcb141", "size": 1693625, "path": "media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "4c84898aed318ecd95cd83b9d459daea4ca54b8963dedad94016c4ace6718e58", "size": 1713057, "path": "media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "38d5fa28d764eeaa07d1aec6e42f028f7e27173f4955f7b3bc8d442c5d180f5f", "size": 1480591, "path": "media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "8da3ce21128658fb6c1e43d103b5b8b0599189e6bf51e3391208e98a209976cd", "size": 410, "artifact_path": "wandb-client-artifact://rwpxtc2wryag22jp57pdhnh73q9n3i5z3txqyz4gk57b7is0wxwlfij07qo7uhz2x3dzxyf3n8kg9f0o8r0gtcnrrd5jd6lyzkfl7qqto8su6v8b89q0karz806dzd6a/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qg0l02qta8eya592hofc6w9ip9lbv97v4rhnyva2fc0s6s3xwud0agegr45yhv95h1mizlum5y6jvcelebu8kxgkycwwt7qsr6zxdahg8pi4iw0ix69c3vwg4ks3w9t2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 14, "total_num_episodes": 40, "total_success_rate": 0.35, "total_success_rate_lb_0.01": 0.19708785420753988, "total_success_rate_ub_0.01": 0.5364554226325556, "total_success_rate_lb_0.025": 0.21984018391013505, "total_success_rate_ub_0.025": 0.49576589465225984, "total_success_rate_lb_0.05": 0.2465912141412675, "total_success_rate_ub_0.05": 0.47091083974227943, "total_success_rate_lb_0.1": 0.24901367356432502, "total_success_rate_ub_0.1": 0.45518821851315283, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3dee92f84846db72ce3825d46975741ee91ce041a8de6f2e101efd5d9a8e8fa4", "size": 512, "artifact_path": "wandb-client-artifact://k7g3r19u575nm9x336wwsvjn62rq0sk0detc8lqajw123rqn1ex4yae0viv623v9o20dphpd5meetctr93r0eybkeacbhvnqs2pb8erik9yux2xo3ecnfkhsgw4po7hx/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://qxszwu8k7fuxn3epgl7o9c4pncof6r9a3hr96waov4thx15m2mc0xr297jis7uja2bsjrzsen4kl79r0vb0jspicrkxb7zfua9ufa6f8l2fy927nakm9dl6crx2a6hp1:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1110}, "eval/total_success_rate": 0.3499999940395355, "eval/total_success_rate_lb_0.01": 0.1970878541469574, "eval/total_success_rate_ub_0.01": 0.5364553928375244, "eval/total_success_rate_lb_0.025": 0.2198401838541031, "eval/total_success_rate_ub_0.025": 0.49576589465141296, "eval/total_success_rate_lb_0.05": 0.2465912103652954, "eval/total_success_rate_ub_0.05": 0.47091084718704224, "eval/total_success_rate_lb_0.1": 0.24901367723941803, "eval/total_success_rate_ub_0.1": 0.45518821477890015} \ No newline at end of file diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug-internal.log b/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..faa46b32003400abf76ed956126f58382d3d5215 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug-internal.log @@ -0,0 +1,2321 @@ +2025-01-18 19:30:35,313 INFO StreamThr :2561591 [internal.py:wandb_internal():85] W&B internal server running at pid: 2561591, started at: 2025-01-18 19:30:35.312670 +2025-01-18 19:30:35,314 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status +2025-01-18 19:30:35,324 INFO WriterThread:2561591 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb +2025-01-18 19:30:35,325 DEBUG SenderThread:2561591 [sender.py:send():379] send: header +2025-01-18 19:30:35,334 DEBUG SenderThread:2561591 [sender.py:send():379] send: run +2025-01-18 19:30:35,573 INFO SenderThread:2561591 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files +2025-01-18 19:30:35,573 INFO SenderThread:2561591 [sender.py:_start_run_threads():1188] run started: 138tidw7 with start time 1737246635.313005 +2025-01-18 19:30:35,592 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: check_version +2025-01-18 19:30:35,592 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: check_version +2025-01-18 19:30:35,662 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: run_start +2025-01-18 19:30:35,667 DEBUG HandlerThread:2561591 [system_info.py:__init__():26] System info init +2025-01-18 19:30:35,667 DEBUG HandlerThread:2561591 [system_info.py:__init__():41] System info init done +2025-01-18 19:30:35,667 INFO HandlerThread:2561591 [system_monitor.py:start():194] Starting system monitor +2025-01-18 19:30:35,667 INFO SystemMonitor:2561591 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-18 19:30:35,667 INFO HandlerThread:2561591 [system_monitor.py:probe():214] Collecting system info +2025-01-18 19:30:35,668 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started cpu monitoring +2025-01-18 19:30:35,668 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started disk monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started gpu monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started memory monitoring +2025-01-18 19:30:35,669 INFO SystemMonitor:2561591 [interfaces.py:start():188] Started network monitoring +2025-01-18 19:30:35,680 DEBUG HandlerThread:2561591 [system_info.py:probe():152] Probing system +2025-01-18 19:30:35,682 DEBUG HandlerThread:2561591 [system_info.py:_probe_git():137] Probing git +2025-01-18 19:30:35,686 DEBUG HandlerThread:2561591 [system_info.py:_probe_git():145] Probing git done +2025-01-18 19:30:35,686 DEBUG HandlerThread:2561591 [system_info.py:probe():200] Probing system done +2025-01-18 19:30:35,687 DEBUG HandlerThread:2561591 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-19T00:30:35.680913', 'startedAt': '2025-01-19T00:30:35.299465', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3565.9246249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4520.276, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4515.837, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 550.1006927490234}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-18 19:30:35,687 INFO HandlerThread:2561591 [system_monitor.py:probe():224] Finished collecting system info +2025-01-18 19:30:35,687 INFO HandlerThread:2561591 [system_monitor.py:probe():227] Publishing system info +2025-01-18 19:30:35,687 DEBUG HandlerThread:2561591 [system_info.py:_save_code():44] Saving code +2025-01-18 19:30:35,692 DEBUG HandlerThread:2561591 [system_info.py:_save_code():67] Saving code done +2025-01-18 19:30:35,693 DEBUG HandlerThread:2561591 [system_info.py:_save_patches():84] Saving git patches +2025-01-18 19:30:35,787 DEBUG HandlerThread:2561591 [system_info.py:_save_patches():126] Saving git patches done +2025-01-18 19:30:35,788 INFO HandlerThread:2561591 [system_monitor.py:probe():229] Finished publishing system info +2025-01-18 19:30:35,791 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-18 19:30:35,791 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-18 19:30:35,792 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-18 19:30:36,024 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: python_packages +2025-01-18 19:30:36,024 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: python_packages +2025-01-18 19:30:36,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:30:36,027 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:30:36,049 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/2l4l5vgf-diff.patch +2025-01-18 19:30:36,051 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/wxd025z5-wandb-metadata.json +2025-01-18 19:30:36,052 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/a313lqel-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:30:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:36,069 INFO HandlerThread:2561591 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/tb" + save: true +} +control { + relay_id: "d6a45ccaf7c14b3aa590c38ecc0ce170" +} +_info { + stream_id: "138tidw7" +} + +2025-01-18 19:30:36,069 DEBUG HandlerThread:2561591 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-18 19:30:36,084 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/ofmwlwa8-code/FISH/eval_robot.py +2025-01-18 19:30:36,168 DEBUG SenderThread:2561591 [sender.py:send():379] send: telemetry +2025-01-18 19:30:36,168 DEBUG SenderThread:2561591 [sender.py:send():379] send: tbrecord +2025-01-18 19:30:36,169 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:30:36,169 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 with policy live +2025-01-18 19:30:36,354 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/sr3ob2h2-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt +2025-01-18 19:30:36,575 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json +2025-01-18 19:30:36,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code +2025-01-18 19:30:36,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH +2025-01-18 19:30:37,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:38,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:38,574 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:39,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:40,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:41,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:41,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:42,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:43,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:44,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:44,487 DEBUG SenderThread:2561591 [sender.py:send():379] send: config +2025-01-18 19:30:45,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:46,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:46,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:46,576 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:47,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:48,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:49,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:50,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:50,577 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:51,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:30:51,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:30:51,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:51,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:52,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:53,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:54,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:54,579 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:30:55,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:56,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:57,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:30:57,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:58,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:30:59,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:00,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:01,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:02,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:02,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:03,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:04,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:05,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:06,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:06,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:06,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:07,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:07,133 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:07,582 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:31:08,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:09,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:10,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:11,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:12,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:13,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:13,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:14,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:15,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:16,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:17,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:18,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:18,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:19,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:20,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:20,586 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:21,025 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:21,025 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:21,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:22,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:23,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:23,097 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:24,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:24,587 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:25,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:26,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:27,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:28,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:29,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:29,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:30,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:31,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:32,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:33,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:34,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:34,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:35,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:35,670 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-18 19:31:35,670 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:31:36,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:36,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:37,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:38,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:39,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:39,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:40,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:41,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:42,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:43,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:44,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:45,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:45,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:46,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:47,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:48,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:49,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:49,734 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:31:49,734 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:31:49,736 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 with policy now +2025-01-18 19:31:49,736 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:31:49,737 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:31:49,737 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:31:49,992 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/x2zq8vpn-media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:50,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:50,594 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:50,595 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:31:50,595 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media +2025-01-18 19:31:50,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:31:50,743 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:50,868 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jz2zp5au-media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:31:51,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:31:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:31:51,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:52,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:53,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:54,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:54,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:55,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:56,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:31:56,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:56,596 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:31:57,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:58,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:31:59,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:00,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:01,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:01,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:01,598 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:01,849 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/nxakqe0x-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:02,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:02,599 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:03,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:04,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:04,599 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:05,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:05,671 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:32:06,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:06,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:06,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:06,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:06,600 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:07,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:08,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:08,601 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:09,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:10,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:11,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:12,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:12,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:13,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:14,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:15,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:16,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:17,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:18,030 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:18,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:19,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:20,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:21,026 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:21,026 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:21,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:22,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:22,606 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:23,042 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:23,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:24,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:25,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:26,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:27,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:28,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:28,599 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:29,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:30,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:30,608 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:31,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:32,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:33,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:33,969 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:34,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:34,217 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:32:34,218 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 with policy now +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:32:34,218 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:32:34,218 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:32:34,466 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/kmrnxouo-media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:34,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:32:34,884 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/ih4zm23x-media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:32:35,048 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:35,671 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:32:36,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:36,027 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:36,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:36,610 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:37,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:38,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:38,611 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:39,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:39,785 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:40,054 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:41,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:42,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:42,612 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:43,056 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:44,057 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:45,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:45,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:46,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:47,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:48,063 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:49,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:50,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:51,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:32:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:32:51,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:51,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:52,064 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:52,510 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:32:52,510 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:32:52,511 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:52,616 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:32:53,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:54,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:54,617 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:55,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:56,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:56,547 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:32:56,618 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:57,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:58,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:32:58,618 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:32:59,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:00,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:01,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:02,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:02,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:03,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:04,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:05,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:05,672 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:33:06,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:06,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:06,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:07,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:07,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:08,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:09,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:10,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:11,066 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:12,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:13,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:13,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:14,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:15,069 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:16,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:17,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:18,041 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:18,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:19,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:20,070 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:21,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:21,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:21,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:22,071 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:22,626 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:23,071 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:23,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:24,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:24,942 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:33:24,943 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 with policy now +2025-01-18 19:33:24,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:33:24,946 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:33:24,946 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:33:24,946 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:33:25,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:25,205 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/1r0tvj7i-media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:25,627 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:33:25,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:25,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:33:25,891 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3wqpggxx-media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:33:26,079 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:26,628 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:27,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:28,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:28,286 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:29,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:30,078 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:30,629 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:31,105 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:32,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:32,630 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:33,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:34,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:34,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:35,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:35,672 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:33:36,034 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:36,034 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:36,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:37,081 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:37,631 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:33:37,876 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3ktxe6xs-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:33:38,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:38,345 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:33:38,345 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:33:38,345 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:33:38,632 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:33:38,632 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:39,083 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:39,601 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:40,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:40,633 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:41,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:42,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:42,633 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:43,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:44,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:44,634 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:33:44,948 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:45,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:46,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:47,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:48,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:49,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:50,046 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:50,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:51,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:33:51,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:33:51,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:52,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:53,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:54,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:55,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:55,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:33:56,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:57,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:58,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:33:59,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:00,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:01,045 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:01,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:02,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:03,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:04,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:05,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:05,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:34:06,028 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:06,028 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:06,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:06,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:07,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:08,095 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:09,095 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:10,104 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:10,637 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:34:10,637 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 with policy now +2025-01-18 19:34:10,641 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:34:10,642 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:10,642 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:10,642 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:10,643 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:34:10,925 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/l5wz1cf1-media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:11,117 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:11,195 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/868y88g3-media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:34:12,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:12,116 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:12,644 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:13,116 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:14,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:15,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:16,118 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:16,645 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:17,106 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:17,993 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:18,109 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:18,646 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:19,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:20,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:21,032 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:21,032 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:21,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:22,115 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:23,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:23,142 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:24,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:25,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:25,402 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:25,403 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:25,403 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:25,648 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:25,648 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:34:26,123 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:26,649 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:27,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:28,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:28,497 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:28,649 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:29,130 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:30,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:30,650 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:31,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:32,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:32,651 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:33,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:34,051 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:34,119 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:35,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:35,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:34:36,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:36,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:36,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:37,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:38,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:39,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:39,174 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:40,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:41,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:42,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:43,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:44,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:45,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:45,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:46,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:47,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:48,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:49,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:50,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:50,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:51,029 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:34:51,029 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:34:51,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:52,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:53,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:54,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:55,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:55,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:34:56,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:57,122 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:58,137 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:34:58,385 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:34:58,385 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 with policy now +2025-01-18 19:34:58,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:34:58,388 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:34:58,388 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:34:58,389 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:34:58,629 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zwzzhutz-media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:58,660 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:34:58,660 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:34:58,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:58,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:34:58,916 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/g1xudixv-media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:34:59,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:00,146 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:00,426 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:00,661 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:01,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:02,143 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:02,662 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:03,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:04,147 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:05,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:05,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:05,673 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:35:06,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:06,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:06,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:06,663 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:06,663 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:35:06,900 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/a5qhx72n-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:35:07,148 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:07,462 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:35:07,462 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:35:07,464 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:35:07,664 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:35:08,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:08,664 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:09,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:10,159 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:10,505 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:11,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:12,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:12,665 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:13,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:14,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:15,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:16,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:16,150 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:17,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:18,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:19,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:20,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:21,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:21,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:21,139 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:21,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:22,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:23,151 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:24,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:25,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:26,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:26,670 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:27,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:27,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:28,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:29,152 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:30,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:31,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:32,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:32,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:32,673 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:33,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:34,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:35,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:35,674 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:35:36,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:36,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:36,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:37,153 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:38,110 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:38,157 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:39,159 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:39,184 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:35:39,184 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 with policy now +2025-01-18 19:35:39,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:35:39,225 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:35:39,226 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:35:39,226 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:35:39,431 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/g79dgf1j-media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:39,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:35:39,920 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/uni7awtz-media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:35:40,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:40,676 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:41,172 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:42,173 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:43,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:43,669 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:44,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:44,678 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:45,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:46,203 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:46,679 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:35:47,177 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:48,186 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:49,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:49,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:50,184 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:51,036 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:35:51,036 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:35:51,196 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:52,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:53,190 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:54,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:35:54,196 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:55,202 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:56,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:57,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:58,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:35:59,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:00,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:00,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:01,219 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:01,529 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:01,530 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:01,530 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:01,684 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:01,684 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:02,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:02,685 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:03,228 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:04,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:05,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:05,674 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:36:05,675 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:06,050 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:06,050 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:06,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:06,686 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:07,211 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:08,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:08,687 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:09,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:10,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:11,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:11,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:12,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:13,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:14,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:15,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:16,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:16,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:17,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:18,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:19,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:20,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:20,691 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:21,033 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:21,033 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:21,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:21,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:22,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:23,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:24,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:24,693 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:25,215 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:26,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:26,693 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:27,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:27,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:28,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:29,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:30,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:31,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:32,216 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:32,902 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:33,235 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:33,987 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:36:33,987 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 with policy now +2025-01-18 19:36:33,996 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:36:33,996 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:33,996 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:33,997 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:34,219 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:34,238 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/65xpujy0-media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:34,696 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:34,697 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:36:34,986 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/qcrnyxba-media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:36:35,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:35,675 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:36:36,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:36,058 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:36,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:37,233 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:38,238 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:38,277 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:38,698 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:39,234 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:40,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:40,578 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:36:40,579 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:36:40,579 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:36:40,699 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:36:40,699 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:40,945 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/3p1t47or-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:36:41,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:42,245 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:42,700 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:43,240 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:43,621 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:44,254 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:45,258 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:46,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:46,701 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:47,261 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:48,262 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:48,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:49,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:50,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:50,702 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:51,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:36:51,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:36:51,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:52,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:53,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:54,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:54,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:36:54,704 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:36:55,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:56,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:57,249 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:58,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:36:59,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:00,073 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:00,250 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:01,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:02,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:03,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:04,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:05,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:05,675 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:37:05,676 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:06,037 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:06,037 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:06,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:07,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:08,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:09,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:10,251 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:11,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:11,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:12,254 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:13,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:14,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:15,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:16,072 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:16,252 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:17,271 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:17,696 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:37:17,696 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 with policy now +2025-01-18 19:37:17,700 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:37:17,701 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:37:17,701 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:37:17,701 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:17,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:37:17,985 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/m4je42ln-media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:18,272 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/8hsb26w0-media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:37:18,285 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:18,713 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:19,268 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:20,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:21,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:21,059 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:21,189 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:21,284 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:22,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:22,714 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:23,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:24,275 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:25,276 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:26,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:26,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:26,635 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:37:26,635 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:37:26,635 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:26,716 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:37:27,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:28,292 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:28,717 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:29,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:30,321 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:31,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:32,104 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:32,306 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:32,718 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:33,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:34,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:35,312 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:35,676 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:37:36,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:36,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:36,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:36,720 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:37,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:37,752 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:38,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:38,721 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:39,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:40,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:41,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:42,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:43,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:43,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:44,301 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:44,723 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:37:45,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:46,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:47,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:48,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:49,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:49,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:50,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:51,038 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:37:51,038 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:37:51,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:52,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:53,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:54,140 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:37:54,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:55,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:56,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:57,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:58,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:37:59,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:00,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:00,306 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:01,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:02,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:03,305 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:04,319 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:04,538 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:38:04,538 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 with policy now +2025-01-18 19:38:04,545 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:38:04,545 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:38:04,545 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:38:04,545 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:04,731 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:04,732 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:38:04,811 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7azmvai9-media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:05,075 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/mugxa2fy-media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:38:05,324 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:05,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:05,676 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:38:06,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:06,081 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:06,328 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:06,732 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:07,342 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:08,344 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:08,733 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:09,350 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:10,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:11,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:11,332 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:12,339 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:12,735 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:13,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:14,357 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:15,383 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:16,090 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:16,348 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:17,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:18,359 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:19,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:20,362 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:21,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:21,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:21,125 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:21,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:22,354 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:23,355 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:24,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:25,378 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:26,385 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:27,108 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:27,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:28,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:28,712 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:38:28,712 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:38:28,712 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:28,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:38:28,977 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/4pcu0qo7-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:38:29,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:30,397 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:30,741 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:31,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:32,121 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:32,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:33,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:34,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:34,743 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:38:35,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:35,677 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:38:36,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:36,043 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:36,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:37,182 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:37,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:38,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:39,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:40,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:41,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:42,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:43,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:43,372 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:44,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:45,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:46,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:47,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:48,092 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:48,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:49,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:50,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:51,043 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:38:51,043 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:38:51,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:52,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:53,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:53,373 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:54,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:55,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:56,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:57,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:58,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:38:58,470 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:38:59,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:00,374 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:00,753 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:01,377 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:01,622 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:39:01,622 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 with policy now +2025-01-18 19:39:01,643 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:39:01,644 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:01,644 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:01,646 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:01,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:01,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:01,755 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:39:01,878 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jtoz3yny-media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:02,133 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/i0cb5oxv-media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:39:02,381 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:02,754 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:03,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:03,708 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:04,402 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:05,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:05,677 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:39:06,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:06,048 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:06,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:06,756 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:07,410 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:08,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:08,757 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:09,221 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:09,412 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:10,407 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:11,399 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:12,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:12,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:12,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:39:12,764 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:12,764 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:12,765 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:13,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:13,759 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:14,411 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:14,760 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:15,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:15,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:16,412 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:16,760 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:17,423 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:18,441 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:19,442 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:20,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:20,448 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:21,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:21,044 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:21,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:22,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:22,762 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:23,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:24,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:25,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:25,878 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:26,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:27,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:28,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:29,436 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:30,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:31,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:31,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:32,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:33,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:34,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:35,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:35,678 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:39:36,044 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:36,044 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:36,114 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:36,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:37,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:38,437 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:38,769 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:39,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:40,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:41,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:41,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:42,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:42,771 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:43,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:44,438 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:45,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:46,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:47,107 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:47,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:48,439 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:49,448 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:49,719 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:39:49,719 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 with policy now +2025-01-18 19:39:49,747 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:39:49,747 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:39:49,748 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:39:49,748 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:39:49,774 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:39:49,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:49,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:39:49,987 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/xbabfegw-media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:50,242 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zolk9upm-media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:39:50,472 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:50,775 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:51,075 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:39:51,075 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:39:51,506 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:52,144 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:52,481 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:53,478 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:54,478 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:54,777 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:55,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:56,474 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:57,469 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:39:57,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:58,498 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:39:58,780 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:39:59,494 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:00,510 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:01,527 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:02,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:03,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:03,542 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:04,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:05,544 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:05,678 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:40:06,086 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:06,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:06,530 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:07,530 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:08,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:08,528 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:08,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:08,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:40:08,833 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:40:08,833 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:40:08,836 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:40:09,033 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/f48blnw7-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:40:09,537 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:09,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:40:10,535 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:10,784 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:11,543 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:12,563 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:13,536 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:13,617 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:14,566 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:14,786 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:15,617 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:16,597 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:17,612 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:18,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:19,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:19,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:20,585 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:20,788 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:21,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:21,047 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:21,585 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:22,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:22,789 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:23,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:24,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:24,586 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:25,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:26,587 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:27,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:28,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:29,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:30,148 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:30,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:31,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:32,588 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:33,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:34,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:35,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:35,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:35,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:40:36,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:36,047 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:36,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:37,589 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:38,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:39,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:40,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:40,595 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:41,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:42,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:43,590 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:44,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:45,591 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:46,149 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:46,592 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:47,603 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:48,354 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:40:48,354 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 with policy now +2025-01-18 19:40:48,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:40:48,363 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:40:48,363 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:40:48,363 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:40:48,614 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:48,616 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/z9ypi00b-media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:48,799 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:40:48,799 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:48,800 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:48,800 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:40:49,072 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/n10zum27-media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:40:49,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:50,603 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:50,803 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:51,065 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:40:51,065 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:40:51,214 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:51,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:52,630 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:52,804 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:53,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:54,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:55,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:56,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:56,806 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:40:57,189 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:40:57,646 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:58,650 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:40:59,629 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:00,674 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:01,644 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:02,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:02,654 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:02,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:02,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:41:02,896 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:41:02,897 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:41:02,897 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:41:03,676 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:03,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:41:04,663 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:04,809 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:05,655 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:05,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:41:06,058 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:06,058 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:06,659 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:07,488 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:07,665 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:08,654 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:08,811 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:09,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:10,685 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:11,699 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:12,683 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:13,181 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:13,685 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:14,704 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:14,813 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:15,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:16,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:16,814 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:17,678 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:18,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:19,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:19,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:20,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:21,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:21,049 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:21,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:22,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:23,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:24,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:25,180 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:25,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:26,679 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:27,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:28,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:29,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:30,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:31,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:31,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:32,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:33,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:34,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:35,679 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:41:35,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:36,049 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:36,049 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:36,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:36,680 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:37,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:38,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:39,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:40,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:41,179 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:41,681 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:42,698 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:42,824 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:42,866 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:41:42,866 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 with policy now +2025-01-18 19:41:42,878 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:41:42,878 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:41:42,878 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:41:42,879 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:41:43,121 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7uoavehf-media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:43,703 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:43,825 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:41:44,124 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/k57q4cig-media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:41:44,726 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:44,826 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:45,729 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:46,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:46,728 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:47,720 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:48,742 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:48,828 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:49,713 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:50,722 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:51,052 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:41:51,052 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:41:51,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:51,721 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:51,829 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:41:52,718 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:53,727 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:54,735 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:55,736 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:56,754 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:57,208 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:41:57,743 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:58,739 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:41:59,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:00,788 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:01,737 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:02,225 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:02,756 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:02,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:02,971 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:02,971 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:02,972 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:03,738 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:03,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:03,834 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:04,750 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:05,680 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:42:05,772 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:05,835 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:06,063 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:06,064 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:06,759 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:07,621 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:07,780 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:08,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:09,774 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:09,837 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:10,792 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:11,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:12,773 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:13,203 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:13,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:14,791 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:15,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:15,839 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:16,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:17,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:17,840 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:18,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:19,194 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:19,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:20,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:21,055 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:21,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:21,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:21,842 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:22,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:23,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:23,843 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:24,525 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:24,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:25,779 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:26,780 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:27,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:28,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:29,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:29,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:30,781 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:31,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:31,846 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:32,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:33,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:34,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:35,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:35,680 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:42:35,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:36,056 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:36,056 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:36,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:37,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:38,782 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:39,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:40,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:41,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:41,783 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:42,793 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:42,980 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:42:42,980 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 with policy now +2025-01-18 19:42:42,989 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:42:42,990 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:42,990 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:42,990 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:43,229 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/doq2mkg2-media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:43,823 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:43,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:42:44,136 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7z0vlu8s-media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:42:44,808 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:44,851 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:45,819 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:45,852 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:46,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:46,805 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:47,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:47,852 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:48,840 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:49,808 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:49,853 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:50,838 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:51,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:42:51,104 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:42:51,230 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:51,833 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:51,854 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:42:52,837 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:53,833 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:54,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:55,840 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:56,856 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:57,228 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:42:57,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:58,835 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:58,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:59,040 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:42:59,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:42:59,040 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:42:59,118 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7jlg2hev-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:42:59,856 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:42:59,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:42:59,858 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:00,907 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:01,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:01,859 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:02,751 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:02,855 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:03,861 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:04,893 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:05,681 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:43:05,860 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:05,907 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:06,085 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:06,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:06,892 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:07,893 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:08,226 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:08,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:09,947 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:10,928 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:11,927 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:12,913 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:13,491 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:13,864 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:13,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:14,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:15,864 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:15,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:16,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:17,865 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:17,915 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:18,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:19,361 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:19,916 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:20,916 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:21,059 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:21,060 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:21,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:22,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:23,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:24,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:25,213 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:25,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:26,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:27,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:28,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:29,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:30,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:31,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:31,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:32,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:33,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:34,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:35,681 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:43:35,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:36,061 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:36,061 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:36,212 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:36,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:37,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:38,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:39,918 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:40,932 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:41,217 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:41,510 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:43:41,510 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 with policy now +2025-01-18 19:43:41,558 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:43:41,559 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:43:41,559 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:43:41,559 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:43:41,757 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/vks9ezhq-media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:41,875 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:41,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:43:41,935 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:42,143 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jsu0qqf7-media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:43:42,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:42,930 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:43,876 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:43,936 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:44,946 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:45,933 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:46,227 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:46,936 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:47,878 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:47,944 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:48,930 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:49,879 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:49,970 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:50,945 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:51,087 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:43:51,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:43:51,259 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:51,955 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:52,953 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:53,959 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:54,105 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:43:54,105 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:43:54,105 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:43:54,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:43:54,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:43:54,954 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:55,881 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:55,948 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:56,280 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:43:56,950 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:57,882 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:57,959 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:58,965 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:43:59,883 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:43:59,969 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:00,980 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:01,884 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:02,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:02,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:02,979 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:03,978 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:04,974 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:05,682 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:44:05,986 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:06,105 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:06,106 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:07,039 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:07,242 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:08,027 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:09,001 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:09,887 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:10,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:11,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:11,888 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:12,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:13,002 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:13,240 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:14,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:15,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:16,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:17,004 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:18,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:19,003 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:19,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:20,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:21,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:21,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:21,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:22,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:23,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:24,007 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:24,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:25,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:26,008 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:27,010 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:28,009 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:29,010 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:29,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:30,011 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:31,011 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:32,013 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:33,013 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:34,014 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:35,017 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:35,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:35,682 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:44:36,015 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:36,067 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:36,067 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:37,046 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:38,031 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:38,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:44:38,089 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:44:38,090 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 with policy now +2025-01-18 19:44:38,090 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:44:38,090 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:44:38,090 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:44:38,348 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/bhfc5gf8-media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:38,899 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:38,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:44:39,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:39,192 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/svdgsjlo-media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:44:39,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:40,053 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:40,900 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:41,048 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:41,222 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:42,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:43,047 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:43,902 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:44,112 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:45,082 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:46,079 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:46,257 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:46,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:44:47,062 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:47,172 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:44:47,172 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:44:47,184 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:44:47,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:44:47,903 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:48,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:49,089 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:49,904 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:50,113 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:51,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:44:51,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:44:51,120 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:52,124 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:52,248 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:53,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:53,907 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:44:54,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:55,138 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:56,200 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:57,170 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:57,265 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:44:58,195 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:44:59,183 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:00,191 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:01,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:02,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:02,939 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:03,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:03,911 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:04,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:05,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:05,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:45:06,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:06,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:06,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:07,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:08,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:08,246 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:09,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:10,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:11,166 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:11,915 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:12,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:13,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:13,317 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:13,916 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:14,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:15,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:16,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:17,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:17,918 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:18,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:18,471 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:19,167 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:19,919 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:20,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:21,068 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:21,068 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:21,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:21,920 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:22,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:23,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:24,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:24,246 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:25,168 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:26,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:27,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:28,169 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:29,178 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:29,598 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:29,923 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:30,033 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:45:30,033 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 with policy now +2025-01-18 19:45:30,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:45:30,041 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:45:30,041 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:45:30,041 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:45:30,181 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:30,295 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/mhbibk0n-media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:30,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:45:31,209 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:31,224 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/hx5ju56f-media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:45:31,924 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:32,205 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:33,264 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:34,241 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:35,239 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:35,593 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:35,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:45:35,926 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:36,094 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:36,094 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:36,268 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:37,230 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:38,236 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:39,279 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:39,928 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:40,298 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:40,974 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:41,318 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:42,310 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:43,295 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:44,240 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:45:44,241 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:45:44,241 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:45:44,300 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:44,930 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:45:44,930 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:45:45,176 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/vwx6wx3m-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:45:45,307 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:45,931 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:46,261 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:46,333 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:47,335 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:47,931 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:48,340 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:49,351 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:50,358 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:51,093 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:45:51,093 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:45:51,295 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:51,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:51,933 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:45:52,369 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:53,394 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:54,380 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:55,389 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:56,387 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:57,292 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:45:57,397 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:58,432 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:45:59,452 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:00,417 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:01,400 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:02,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:03,098 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:03,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:03,938 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:04,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:05,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:05,683 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:46:06,076 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:06,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:06,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:07,401 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:08,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:08,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:09,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:10,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:11,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:12,403 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:13,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:14,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:14,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:15,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:16,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:17,404 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:18,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:19,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:19,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:20,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:21,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:21,077 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:21,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:22,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:23,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:24,256 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:24,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:25,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:26,405 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:27,406 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:28,406 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:29,426 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:29,671 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:30,117 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:46:30,117 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 with policy now +2025-01-18 19:46:30,127 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:46:30,128 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:46:30,128 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:46:30,128 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:46:30,378 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/slp670pq-media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:30,454 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:30,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:46:31,251 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/8pwr0wse-media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:46:31,457 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:31,950 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:32,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:33,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:34,469 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:34,861 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:35,485 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:35,684 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:46:35,952 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:36,091 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:36,091 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:36,493 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:37,470 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:38,516 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:39,482 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:39,954 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:40,293 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:40,502 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:41,560 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:42,308 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:46:42,308 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:46:42,309 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:46:42,510 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:42,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:46:42,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:46:43,519 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:43,955 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:44,536 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:45,540 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:45,717 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:45,956 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:46,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:47,533 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:48,565 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:49,547 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:49,958 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:46:50,569 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:51,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:46:51,103 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:46:51,171 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:51,561 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:52,564 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:53,581 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:54,596 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:55,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:56,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:46:56,600 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:57,597 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:58,649 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:46:59,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:00,626 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:01,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:01,963 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:02,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:02,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:03,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:03,964 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:04,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:05,605 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:05,684 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:47:06,077 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:06,078 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:06,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:07,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:07,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:08,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:09,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:10,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:11,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:12,285 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:12,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:13,606 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:14,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:15,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:16,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:17,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:18,278 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:18,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:19,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:20,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:21,078 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:21,078 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:21,607 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:22,610 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:23,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:23,917 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:23,972 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:24,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:25,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:26,609 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:27,608 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:28,657 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:28,957 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:29,146 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:47:29,146 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 with policy now +2025-01-18 19:47:29,176 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:47:29,176 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:47:29,177 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:47:29,177 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:47:29,407 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7wmngbix-media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:29,662 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:29,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:47:30,272 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/w9q0pag3-media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:47:30,663 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:30,976 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:31,634 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:31,977 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:32,645 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:33,667 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:34,746 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:34,957 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:35,685 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:47:35,739 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:35,979 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:36,103 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:36,103 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:36,703 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:37,706 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:37,980 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:38,733 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:39,733 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:40,341 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:40,376 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:47:40,376 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:47:40,377 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:47:40,786 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:40,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:47:40,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:47:41,738 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:41,981 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:42,746 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:43,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:43,982 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:44,761 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:45,737 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:46,270 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:46,758 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:47,762 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:47,984 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:48,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:49,785 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:50,756 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:51,127 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:47:51,127 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:47:51,355 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:51,789 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:52,803 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:53,778 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:54,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:55,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:55,987 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:56,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:57,302 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:47:57,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:57,989 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:47:58,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:47:59,764 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:00,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:01,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:02,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:02,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:03,765 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:03,991 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:04,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:05,685 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:48:05,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:06,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:06,080 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:06,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:07,303 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:07,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:08,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:09,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:10,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:11,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:12,304 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:12,766 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:13,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:14,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:15,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:16,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:17,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:17,901 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:18,767 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:19,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:19,999 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:20,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:21,080 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:21,081 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:21,768 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:22,796 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:22,906 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:23,352 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:23,353 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 with policy now +2025-01-18 19:48:23,367 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:23,368 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:23,368 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:23,368 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:23,615 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/6kkt3sa8-media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:23,771 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:24,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:24,002 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval +2025-01-18 19:48:24,278 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/zjr2rw51-media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:48:24,803 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:25,001 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:25,813 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:26,002 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:26,854 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:27,855 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:27,933 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:28,821 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:29,828 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:30,004 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:30,853 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:31,851 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:32,005 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:32,829 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:33,371 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:33,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:34,873 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:35,686 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:48:35,886 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:36,086 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:36,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:36,868 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:37,448 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:37,448 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:37,449 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:37,858 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:38,007 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:38,007 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:38,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:38,913 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:39,269 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:48:39,269 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:48:39,472 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:39,707 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxt9zj5z_ +2025-01-18 19:48:39,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:39,939 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-18 19:48:39,981 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: partial_history +2025-01-18 19:48:40,008 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:40,008 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval +2025-01-18 19:48:40,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table +2025-01-18 19:48:40,081 INFO SenderThread:2561591 [sender.py:send_request_log_artifact():1525] logged artifact run-138tidw7-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTQ1MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA0Mzg3Nw==', 'latestArtifact': None}} +2025-01-18 19:48:40,081 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:40,081 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json with policy now +2025-01-18 19:48:40,082 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:40,082 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,083 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,083 DEBUG SenderThread:2561591 [sender.py:send():379] send: telemetry +2025-01-18 19:48:40,083 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,083 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,084 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,084 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,084 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,084 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,085 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,086 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,086 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,086 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,086 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,087 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,087 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,087 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,088 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,088 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,088 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,089 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:40,089 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: log_artifact +2025-01-18 19:48:40,310 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/7h77vpfn-media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:40,317 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/57l1z76y-media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,330 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/jd2jbmbt-events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:48:40,514 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3s55t847 +2025-01-18 19:48:40,565 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/lslxpsj3-media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:48:40,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:40,927 INFO SenderThread:2561591 [sender.py:send_request_log_artifact():1525] logged artifact run-138tidw7-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTQ2MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MzA0Mzg4MA==', 'latestArtifact': None}} +2025-01-18 19:48:40,927 DEBUG SenderThread:2561591 [sender.py:send():379] send: files +2025-01-18 19:48:40,927 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json with policy now +2025-01-18 19:48:40,927 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:48:40,928 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:48:40,928 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:48:41,009 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:48:41,216 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /tmp/tmp9dnbyfknwandb/txdpo6ti-media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:48:41,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:42,010 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:48:42,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:43,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:44,359 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:44,881 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:45,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:46,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:47,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:48,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:49,882 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:50,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:50,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:51,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:48:51,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:48:51,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:52,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:53,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:54,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:55,360 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:48:55,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:56,883 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:57,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:58,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:48:59,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:00,363 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:00,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:01,020 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:49:01,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:02,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:03,884 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:04,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:05,686 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:49:05,687 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:05,885 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:06,084 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: stop_status +2025-01-18 19:49:06,085 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: stop_status +2025-01-18 19:49:06,222 DEBUG SenderThread:2561591 [sender.py:send():379] send: exit +2025-01-18 19:49:06,222 INFO SenderThread:2561591 [sender.py:send_exit():586] handling exit code: 0 +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:send_exit():588] handling runtime: 1110 +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:06,223 INFO SenderThread:2561591 [sender.py:send_exit():594] send defer +2025-01-18 19:49:06,223 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-18 19:49:06,224 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 1 +2025-01-18 19:49:06,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-18 19:49:06,224 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-18 19:49:06,224 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 2 +2025-01-18 19:49:06,224 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-18 19:49:06,224 INFO HandlerThread:2561591 [system_monitor.py:finish():203] Stopping system monitor +2025-01-18 19:49:06,224 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-18 19:49:06,225 DEBUG SystemMonitor:2561591 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-18 19:49:06,225 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined cpu monitor +2025-01-18 19:49:06,225 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined disk monitor +2025-01-18 19:49:06,275 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined gpu monitor +2025-01-18 19:49:06,275 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined memory monitor +2025-01-18 19:49:06,276 INFO HandlerThread:2561591 [interfaces.py:finish():200] Joined network monitor +2025-01-18 19:49:06,276 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,276 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-18 19:49:06,276 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 3 +2025-01-18 19:49:06,276 DEBUG SenderThread:2561591 [sender.py:send():379] send: stats +2025-01-18 19:49:06,276 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,276 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-18 19:49:06,277 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:06,277 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-18 19:49:06,277 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 4 +2025-01-18 19:49:06,277 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:06,277 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-18 19:49:07,023 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:08,024 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:12,466 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: keepalive +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,467 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-18 19:49:12,467 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 5 +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send():379] send: history +2025-01-18 19:49:12,467 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: summary_record +2025-01-18 19:49:12,468 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,468 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-18 19:49:12,468 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:12,474 DEBUG SenderThread:2561591 [sender.py:send():379] send: summary +2025-01-18 19:49:12,474 INFO SenderThread:2561591 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-18 19:49:12,474 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 6 +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,475 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-18 19:49:12,475 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-18 19:49:12,475 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 7 +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: status_report +2025-01-18 19:49:12,475 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:12,475 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-18 19:49:12,475 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:12,476 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-18 19:49:13,029 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:13,155 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 8 +2025-01-18 19:49:14,905 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:14,905 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:14,905 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-18 19:49:14,905 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-18 19:49:14,905 INFO SenderThread:2561591 [job_builder.py:build():440] Attempting to build job artifact +2025-01-18 19:49:14,908 INFO SenderThread:2561591 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-18 19:49:14,910 INFO SenderThread:2561591 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-18 19:49:14,925 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 9 +2025-01-18 19:49:14,925 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:14,925 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-18 19:49:14,926 DEBUG SenderThread:2561591 [sender.py:send():379] send: artifact +2025-01-18 19:49:15,030 INFO Thread-12 :2561591 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:15,156 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:15,377 INFO wandb-upload_2:2561591 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpauwgphec +2025-01-18 19:49:15,379 INFO wandb-upload_0:2561591 [upload_job.py:push():85] Skipped uploading /tmp/tmptxvvuw2w/wandb-job.json +2025-01-18 19:49:15,527 INFO wandb-upload_3:2561591 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp47q_99wi +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1MTkxOTkxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1MTg4ODMzOQ==', 'versionIndex': 447}}} +2025-01-18 19:49:16,473 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-18 19:49:16,473 INFO SenderThread:2561591 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml config.yaml +2025-01-18 19:49:17,031 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json wandb-summary.json +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 events.out.tfevents.1737246636.leonmkim-ROG-Strix-G15CS-G15CS.2561402.0 +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt requirements.txt +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-metadata.json wandb-metadata.json +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log output.log +2025-01-18 19:49:17,032 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/diff.patch diff.patch +2025-01-18 19:49:17,035 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 media/videos/eval/18_eval_18_4c84898aed318ecd95cd.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 media/videos/eval/14_eval_14_da508fd72acfd8e8c94e.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 media/videos/eval/10_eval_10_b5b221e9eaaa2afeafab.mp4 +2025-01-18 19:49:17,038 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 media/videos/eval/2_eval_2_dcfa4207d555c0dc155d.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 media/videos/eval/19_eval_19_38d5fa28d764eeaa07d1.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 media/videos/eval/7_eval_7_21bd94bd9f2e25c7a138.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 media/videos/eval/11_eval_11_60d95850d1df3e2b9234.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 media/videos/eval/4_eval_4_573f9d77908239cd99f4.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 media/videos/eval/8_eval_8_8ad80c418d3b655ed57a.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 media/videos/eval/17_eval_17_415f21e10d2661428c43.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 media/videos/eval/15_eval_15_f8f145ad74c481c07c43.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 media/videos/eval/12_eval_12_11b1f9ef1d691b07f19a.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 media/videos/eval/1_eval_1_b0aac6515bc4a872c577.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 media/videos/eval/0_eval_0_96591af35a54c45302d0.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 media/videos/eval/6_eval_6_8d85fdaf1b462a9c0774.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 media/videos/eval/9_eval_9_3cc9e9466280c44c4ea9.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 media/videos/eval/13_eval_13_a298f612d50a7b95e221.mp4 +2025-01-18 19:49:17,039 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 media/videos/eval/16_eval_16_4567bcc512ee0512f3ae.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 media/videos/eval/3_eval_3_6854d6b9b0ee461bdf8c.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 media/videos/eval/5_eval_5_8f6b889b8499cd4be78a.mp4 +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json media/table/eval/total_success_rate_ci_21_3dee92f84846db72ce38.table.json +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json media/table/eval/success_rate_ci_20_8da3ce21128658fb6c1e.table.json +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 10 +2025-01-18 19:49:17,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,040 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,040 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-18 19:49:17,040 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,040 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-18 19:49:17,041 INFO SenderThread:2561591 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:49:17,156 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,156 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,211 INFO wandb-upload_0:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/requirements.txt +2025-01-18 19:49:17,230 INFO wandb-upload_2:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/wandb-summary.json +2025-01-18 19:49:17,232 INFO wandb-upload_3:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/output.log +2025-01-18 19:49:17,243 INFO wandb-upload_1:2561591 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/files/config.yaml +2025-01-18 19:49:17,444 INFO Thread-11 (_thread_body):2561591 [sender.py:transition_state():614] send defer: 11 +2025-01-18 19:49:17,444 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,444 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-18 19:49:17,444 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,444 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-18 19:49:17,444 INFO SenderThread:2561591 [file_pusher.py:join():175] waiting for file pusher +2025-01-18 19:49:17,448 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 12 +2025-01-18 19:49:17,449 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,449 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-18 19:49:17,449 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,449 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-18 19:49:17,449 INFO SenderThread:2561591 [file_stream.py:finish():601] file stream finish called +2025-01-18 19:49:17,623 INFO SenderThread:2561591 [file_stream.py:finish():605] file stream finish is done +2025-01-18 19:49:17,623 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 13 +2025-01-18 19:49:17,623 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,623 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:transition_state():614] send defer: 14 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send():379] send: final +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send():379] send: footer +2025-01-18 19:49:17,624 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: defer +2025-01-18 19:49:17,624 INFO HandlerThread:2561591 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-18 19:49:17,624 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: defer +2025-01-18 19:49:17,624 INFO SenderThread:2561591 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: server_info +2025-01-18 19:49:17,625 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-18 19:49:17,625 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,626 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: server_info +2025-01-18 19:49:17,627 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: get_summary +2025-01-18 19:49:17,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-18 19:49:17,628 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-18 19:49:17,687 DEBUG SenderThread:2561591 [sender.py:send_request():406] send_request: poll_exit +2025-01-18 19:49:17,687 INFO MainThread:2561591 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-18 19:49:17,688 INFO MainThread:2561591 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-18 19:49:17,688 INFO MainThread:2561591 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-18 19:49:17,688 DEBUG HandlerThread:2561591 [handler.py:handle_request():158] handle_request: shutdown +2025-01-18 19:49:17,688 INFO HandlerThread:2561591 [handler.py:finish():882] shutting down handler +2025-01-18 19:49:18,626 INFO WriterThread:2561591 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [sender.py:finish():1615] shutting down sender +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [file_pusher.py:finish():169] shutting down file pusher +2025-01-18 19:49:18,687 INFO SenderThread:2561591 [file_pusher.py:join():175] waiting for file pusher diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug.log b/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7f75e1d9af45815900ae1d8a2404cf269a55b6c2 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Configure stats pid to 2561402 +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/settings +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug.log +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/wandb/run-20250118_193035-138tidw7/logs/debug-internal.log +2025-01-18 19:30:35,307 INFO MainThread:2561402 [wandb_init.py:init():569] calling init triggers +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7'} +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():619] starting backend +2025-01-18 19:30:35,308 INFO MainThread:2561402 [wandb_init.py:init():623] setting up manager +2025-01-18 19:30:35,311 INFO MainThread:2561402 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-18 19:30:35,312 INFO MainThread:2561402 [wandb_init.py:init():631] backend started and connected +2025-01-18 19:30:35,324 INFO MainThread:2561402 [wandb_init.py:init():720] updated telemetry +2025-01-18 19:30:35,332 INFO MainThread:2561402 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-18 19:30:35,591 INFO MainThread:2561402 [wandb_run.py:_on_init():2435] communicating current version +2025-01-18 19:30:35,656 INFO MainThread:2561402 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-18 19:30:35,657 INFO MainThread:2561402 [wandb_init.py:init():804] starting run threads in backend +2025-01-18 19:30:36,026 INFO MainThread:2561402 [wandb_run.py:_console_start():2413] atexit reg +2025-01-18 19:30:36,027 INFO MainThread:2561402 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-18 19:30:36,028 INFO MainThread:2561402 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-18 19:30:36,028 INFO MainThread:2561402 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-18 19:30:36,029 INFO MainThread:2561402 [wandb_init.py:init():847] run started, returning control to user process +2025-01-18 19:30:36,029 INFO MainThread:2561402 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/138tidw7/tb, True +2025-01-18 19:30:44,487 INFO MainThread:2561402 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-18 19:49:18,689 WARNING MsgRouterThr:2561402 [router.py:message_loop():77] message_loop has been closed diff --git a/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb b/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d7df0549f357e685d08a4fc04fd096e2face40b9 --- /dev/null +++ b/138tidw7/wandb/run-20250118_193035-138tidw7/run-138tidw7.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f170364412e026ea928be08c96fb52264363e77cb98da19c9601d291ba3e6525 +size 235393 diff --git a/213npkcd/.hydra/config.yaml b/213npkcd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd7a0d9043cd058869b33cb503156d329d779fab --- /dev/null +++ b/213npkcd/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/213npkcd/.hydra/hydra.yaml b/213npkcd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d1791def218dd157b34ff55d2eae0d5120cd574 --- /dev/null +++ b/213npkcd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/213npkcd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/213npkcd/.hydra/overrides.yaml b/213npkcd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/213npkcd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/213npkcd/eval_policy.log b/213npkcd/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..c8e0b27b90bf936a3655f1b8f9df7c6bfa6734ce --- /dev/null +++ b/213npkcd/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-18 19:30:27,691][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 19:30:27,696][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-18 19:30:30,906][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/bn7ul4i2/.hydra/config.yaml b/bn7ul4i2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e --- /dev/null +++ b/bn7ul4i2/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bn7ul4i2/.hydra/hydra.yaml b/bn7ul4i2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b18976dba8149e33b009ddbe102e093ed0cd1b2 --- /dev/null +++ b/bn7ul4i2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bn7ul4i2/.hydra/overrides.yaml b/bn7ul4i2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bn7ul4i2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bn7ul4i2/episode_rosbags/aligned_depth_to_color_K.npy b/bn7ul4i2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bn7ul4i2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bn7ul4i2/episode_rosbags/cam_tf_world.npy b/bn7ul4i2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/bn7ul4i2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/bn7ul4i2/episode_rosbags/color_K.npy b/bn7ul4i2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bn7ul4i2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bn7ul4i2/episode_rosbags/depth_K.npy b/bn7ul4i2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/bn7ul4i2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/bn7ul4i2/episode_rosbags/episode_1_2025-01-21-21-07-52.bag b/bn7ul4i2/episode_rosbags/episode_1_2025-01-21-21-07-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..b4f51a75100ecbea485638fede291b15898185ea --- /dev/null +++ b/bn7ul4i2/episode_rosbags/episode_1_2025-01-21-21-07-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30c045599589a541aead1fa96803730fa4236d12d5501584dbbf779886ea26a +size 1412661451 diff --git a/bn7ul4i2/episode_rosbags/episode_2_2025-01-21-21-08-37.bag b/bn7ul4i2/episode_rosbags/episode_2_2025-01-21-21-08-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..43141a384bf4fb5a67e99049a4087de7b4fc1df7 --- /dev/null +++ b/bn7ul4i2/episode_rosbags/episode_2_2025-01-21-21-08-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a0fe2c8eeada3adbbf39b04c64f4220958eba67e4bc72dd32f5bf7a636f395 +size 1413062993 diff --git a/bn7ul4i2/episode_rosbags/episode_3_2025-01-21-21-09-21.bag b/bn7ul4i2/episode_rosbags/episode_3_2025-01-21-21-09-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..e5c3b0f7fae1ac362f13a45f434d28fcf4dea1f7 --- /dev/null +++ b/bn7ul4i2/episode_rosbags/episode_3_2025-01-21-21-09-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d3b2d23d0379e93ad7c868800674e06ededafc6e649814157e0a428725fa131 +size 1414601518 diff --git a/bn7ul4i2/eval_robot.log b/bn7ul4i2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..49026190808e03cc5c22ba8097d79f104225c432 --- /dev/null +++ b/bn7ul4i2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:06:45,577][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:06:45,580][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/bn7ul4i2/eval_video/0_eval.mp4 b/bn7ul4i2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fdad059172460d96638ea3f1c4b3d1a35b60620 --- /dev/null +++ b/bn7ul4i2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92d3370f0db4ebdd4aaf2172036b99317be720ae0952524d557ebe3c439b662 +size 1171918 diff --git a/bn7ul4i2/eval_video/1_eval.mp4 b/bn7ul4i2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..02e238a1342946ee7d93faaa6d9d2b82cefe9dbb --- /dev/null +++ b/bn7ul4i2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a38d84554f950c6b0497c97ea3f3390c361b7c3f4f4ddab98cbeee49507190cb +size 1185838 diff --git a/bn7ul4i2/eval_video/2_eval.mp4 b/bn7ul4i2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bae28d2ebd4f7cfeb6f68d8fc0d62e4e0c4c422 --- /dev/null +++ b/bn7ul4i2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428c4186f8f49dcdf56485fbf404101bf90ec5646c06f2b4294019741ec3860a +size 1186953 diff --git a/bn7ul4i2/eval_video/3_eval.mp4 b/bn7ul4i2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7d7d03ff15e06ba71309fc6c42b537ffb4670a9 --- /dev/null +++ b/bn7ul4i2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0cf70711762442af0f56c2c14d782a6116c3fda485ea86581a3951c50f3f6cf +size 1126837 diff --git a/bn7ul4i2/eval_video/4_eval.mp4 b/bn7ul4i2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c43ac9f62459bc29210e81346d9a35007c4de30b --- /dev/null +++ b/bn7ul4i2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd715cc078221a229a9dc2bedfce4214f34bb940ca892bb65ed257dc339dc4c3 +size 1175795 diff --git a/bn7ul4i2/tb/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 b/bn7ul4i2/tb/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 new file mode 100644 index 0000000000000000000000000000000000000000..6319894029a298faa3be25ffcf570f2f0cb4849e --- /dev/null +++ b/bn7ul4i2/tb/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:603867abe01b59d5f60685db94359b5e13baefc6a8e6842222e6f903cb0e5b30 +size 6539 diff --git a/bn7ul4i2/wandb/debug-internal.log b/bn7ul4i2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6c2eae2b7bfd7c60c715cb32efd95510a06b4aa8 --- /dev/null +++ b/bn7ul4i2/wandb/debug-internal.log @@ -0,0 +1,768 @@ +2025-01-21 21:06:51,388 INFO StreamThr :2645977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2645977, started at: 2025-01-21 21:06:51.388319 +2025-01-21 21:06:51,389 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:06:51,397 INFO WriterThread:2645977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb +2025-01-21 21:06:51,398 DEBUG SenderThread:2645977 [sender.py:send():379] send: header +2025-01-21 21:06:51,407 DEBUG SenderThread:2645977 [sender.py:send():379] send: run +2025-01-21 21:06:51,716 INFO SenderThread:2645977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files +2025-01-21 21:06:51,717 INFO SenderThread:2645977 [sender.py:_start_run_threads():1188] run started: bn7ul4i2 with start time 1737511611.388322 +2025-01-21 21:06:51,735 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:06:51,735 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:06:51,781 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:06:51,785 DEBUG HandlerThread:2645977 [system_info.py:__init__():26] System info init +2025-01-21 21:06:51,786 DEBUG HandlerThread:2645977 [system_info.py:__init__():41] System info init done +2025-01-21 21:06:51,786 INFO HandlerThread:2645977 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:06:51,786 INFO SystemMonitor:2645977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:06:51,786 INFO HandlerThread:2645977 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:06:51,788 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:06:51,788 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:06:51,799 DEBUG HandlerThread:2645977 [system_info.py:probe():152] Probing system +2025-01-21 21:06:51,801 DEBUG HandlerThread:2645977 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_info.py:probe():200] Probing system done +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:06:51.799700', 'startedAt': '2025-01-22T02:06:51.381170', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.5710000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4497.405, 'min': 800.0, 'max': 4700.0}, {'current': 4499.988, 'min': 800.0, 'max': 4700.0}, {'current': 4496.469, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.983, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 344.01396560668945}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:06:51,806 INFO HandlerThread:2645977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:06:51,806 INFO HandlerThread:2645977 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:06:51,806 DEBUG HandlerThread:2645977 [system_info.py:_save_code():44] Saving code +2025-01-21 21:06:51,807 DEBUG HandlerThread:2645977 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:06:51,807 DEBUG HandlerThread:2645977 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:06:51,869 DEBUG HandlerThread:2645977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:06:51,869 INFO HandlerThread:2645977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:06:51,872 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:06:51,872 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:06:51,872 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:06:51,873 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:06:52,114 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:06:52,114 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:06:52,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:52,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:52,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:52,117 INFO HandlerThread:2645977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/tb" + save: true +} +control { + relay_id: "8fcde4342c68447694e117753e76da0a" +} +_info { + stream_id: "bn7ul4i2" +} + +2025-01-21 21:06:52,117 DEBUG HandlerThread:2645977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:06:52,125 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/u8tdtv45-wandb-metadata.json +2025-01-21 21:06:52,143 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/3pw2oj2i-code/FISH/eval_robot.py +2025-01-21 21:06:52,180 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/dukk9114-diff.patch +2025-01-21 21:06:52,185 DEBUG SenderThread:2645977 [sender.py:send():379] send: telemetry +2025-01-21 21:06:52,185 DEBUG SenderThread:2645977 [sender.py:send():379] send: tbrecord +2025-01-21 21:06:52,186 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:06:52,186 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 with policy live +2025-01-21 21:06:52,388 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/8dovbc8h-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code +2025-01-21 21:06:53,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:54,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:54,717 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:55,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:56,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:56,540 DEBUG SenderThread:2645977 [sender.py:send():379] send: config +2025-01-21 21:06:56,540 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:57,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:58,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:58,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:59,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:00,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:01,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:02,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:02,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:02,720 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:03,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:04,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:05,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:06,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:06,721 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:07,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:07,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:07,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:07,277 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:08,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:08,721 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:09,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:10,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:10,722 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:11,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:12,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:12,723 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:12,913 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:13,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:14,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:14,723 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:15,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:16,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:17,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:18,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:18,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:19,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:20,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:21,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:22,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:22,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:22,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:23,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:23,701 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:24,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:24,726 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:07:25,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:26,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:27,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:28,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:29,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:29,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:30,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:31,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:32,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:33,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:34,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:35,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:35,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:36,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:37,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:37,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:37,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:37,405 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:07:37,406 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 with policy now +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:07:37,407 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:07:37,698 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/7seopqdc-media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:37,729 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:37,729 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media +2025-01-21 21:07:38,008 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/4cu6sdrr-media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:38,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:38,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:39,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:40,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:40,408 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:41,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:42,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:42,731 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:43,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:44,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:44,732 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:45,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:45,737 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:46,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:47,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:48,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:49,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:49,733 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:07:50,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:50,734 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:51,037 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:51,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:51,788 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:07:51,789 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:07:52,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:52,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:52,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:52,735 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:53,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:54,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:54,735 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:55,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:56,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:56,948 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:57,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:58,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:59,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:00,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:01,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:02,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:02,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:03,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:04,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:05,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:06,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:07,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:07,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:07,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:07,256 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:08,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:09,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:10,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:11,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:12,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:13,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:13,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:14,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:15,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:16,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:17,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:18,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:18,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:19,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:20,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:20,743 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:20,974 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:08:20,974 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 with policy now +2025-01-21 21:08:20,974 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:08:20,975 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:08:20,975 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:08:20,975 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:08:21,129 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:21,263 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/gvmll21x-media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:08:21,789 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:08:22,068 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/g0sx2ln3-media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:22,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:22,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:22,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:22,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:23,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:23,277 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:23,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:24,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:25,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:26,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:26,745 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:27,139 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:28,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:28,417 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:28,746 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:29,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:30,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:31,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:32,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:33,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:34,074 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:34,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:34,592 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:08:34,592 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:08:34,593 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:08:34,748 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:08:34,749 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:08:35,038 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/287aceph-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:08:35,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:36,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:36,749 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:37,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:37,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:37,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:38,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:38,750 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:39,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:39,476 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:40,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:40,751 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:41,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:42,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:43,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:44,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:45,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:45,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:46,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:47,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:48,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:49,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:50,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:50,921 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:51,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:51,790 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:08:52,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:52,117 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:52,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:52,755 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:53,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:54,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:55,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:56,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:56,150 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:56,756 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:57,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:58,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:59,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:00,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:00,757 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:01,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:02,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:02,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:03,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:04,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:04,759 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:05,139 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:06,141 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:06,265 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:09:06,266 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:09:06,266 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 with policy now +2025-01-21 21:09:06,266 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:06,267 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:06,267 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:06,550 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/w49ytuch-media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:09:07,102 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/3862svq1-media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:07,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:07,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:07,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:07,257 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:08,155 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:08,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:09,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:10,141 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:11,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:12,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:12,731 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:12,762 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:13,142 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:14,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:14,764 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:15,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:16,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:17,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:17,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:09:18,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:18,352 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:18,352 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:18,353 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:18,353 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:18,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:18,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:19,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:20,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:20,766 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:21,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:21,791 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:09:22,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:22,117 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:22,144 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:22,766 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:23,144 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:24,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:24,261 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:25,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:26,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:27,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:28,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:29,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:30,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:30,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:31,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:32,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:33,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:34,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:35,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:35,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:36,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:37,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:37,118 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:37,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:38,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:39,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:40,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:40,247 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:41,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:42,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:43,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:44,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:45,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:46,129 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:46,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:47,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:48,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:48,776 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:49,152 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:50,033 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:09:50,033 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 with policy now +2025-01-21 21:09:50,036 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:09:50,037 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:50,037 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:50,037 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:50,152 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:50,341 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/fwhbx8yt-media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:50,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:09:51,093 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/wpl0pvln-media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:51,159 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:51,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:51,791 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:09:51,792 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:52,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:52,133 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:52,177 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:52,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:53,164 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:54,163 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:54,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:55,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:56,166 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:57,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:57,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:58,178 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:58,780 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:59,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:00,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:01,180 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:02,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:02,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:02,407 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:02,407 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:02,408 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:02,781 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:02,781 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:03,062 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/y8fiahmw-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:03,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:04,178 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:04,782 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:05,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:06,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:06,783 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:07,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:07,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:07,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:07,204 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:08,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:08,783 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:09,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:10,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:11,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:12,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:12,785 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:13,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:13,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:14,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:15,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:16,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:17,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:18,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:18,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:19,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:20,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:21,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:21,792 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:10:22,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:22,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:22,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:23,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:23,272 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:24,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:25,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:26,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:27,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:28,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:29,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:29,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:30,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:31,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:32,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:33,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:34,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:34,183 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:34,229 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:34,229 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 with policy now +2025-01-21 21:10:34,230 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:34,230 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:34,230 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:34,231 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:34,517 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/7ygne6de-media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:34,792 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:10:35,086 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/kzqyng5g-media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:35,185 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:35,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:36,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:36,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:37,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:37,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:37,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:38,189 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:38,794 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:39,190 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:39,913 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:40,191 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:41,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:42,199 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:42,795 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:43,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:43,456 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:43,456 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:43,457 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:43,609 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:10:43,609 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table +2025-01-21 21:10:43,843 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:44,000 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/4f5g6jxq-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:44,134 INFO wandb-upload_0:2645977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmp6_2_p8 +2025-01-21 21:10:44,192 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:44,246 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:10:44,289 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:44,553 INFO SenderThread:2645977 [sender.py:send_request_log_artifact():1525] logged artifact run-bn7ul4i2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0OTgwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MjAxMQ==', 'latestArtifact': None}} +2025-01-21 21:10:44,553 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:44,553 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-21 21:10:44,553 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,554 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send():379] send: telemetry +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,558 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,558 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,559 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,559 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,560 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,560 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,561 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,561 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,561 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,561 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,562 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,562 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,563 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:10:44,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:44,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval +2025-01-21 21:10:44,882 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/1bptd7ap-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:44,924 INFO wandb-upload_0:2645977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5qfs4pat +2025-01-21 21:10:45,015 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/rrq8xh32-media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:45,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:45,363 INFO SenderThread:2645977 [sender.py:send_request_log_artifact():1525] logged artifact run-bn7ul4i2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0OTgyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MjAxNg==', 'latestArtifact': None}} +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:45,363 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json with policy now +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:45,364 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:45,364 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:45,636 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/ahlofooo-media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:45,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:46,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:46,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:47,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:48,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:49,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:50,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:51,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:51,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:51,792 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:10:52,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:52,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:52,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:53,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:54,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:55,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:56,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:56,281 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:57,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:58,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:59,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:00,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:01,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:02,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:02,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:03,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:04,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:05,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:06,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:07,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:11:07,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:11:07,192 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:07,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:08,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:09,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:10,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:11,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:12,196 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:13,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:13,196 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:13,577 DEBUG SenderThread:2645977 [sender.py:send():379] send: exit +2025-01-21 21:11:13,577 INFO SenderThread:2645977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:11:13,578 INFO SenderThread:2645977 [sender.py:send_exit():588] handling runtime: 261 +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:send_exit():594] send defer +2025-01-21 21:11:13,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,579 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:11:13,579 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:11:13,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,579 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:11:13,580 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,580 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:11:13,580 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:11:13,580 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:11:13,580 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:11:13,580 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:11:13,651 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:11:13,652 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:11:13,652 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,652 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:11:13,652 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:11:13,652 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,652 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:11:13,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:11:13,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:14,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,467 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:11:20,467 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:11:20,467 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,467 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:11:20,468 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:20,468 DEBUG SenderThread:2645977 [sender.py:send():379] send: summary +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:20,472 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,473 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:11:20,473 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,473 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:11:20,473 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:11:20,578 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:20,811 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:23,149 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:11:23,149 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:23,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,150 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:11:23,150 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,150 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:11:23,150 INFO SenderThread:2645977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:11:23,152 INFO SenderThread:2645977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:11:23,154 INFO SenderThread:2645977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:11:23,169 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:11:23,169 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,169 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:11:23,169 DEBUG SenderThread:2645977 [sender.py:send():379] send: artifact +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:11:23,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:11:23,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml config.yaml +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json wandb-summary.json +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt requirements.txt +2025-01-21 21:11:23,813 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:11:23,813 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:11:23,817 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log output.log +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch diff.patch +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:11:23,820 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:23,822 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,822 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:11:23,822 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,822 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:11:23,822 INFO SenderThread:2645977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:11:24,003 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:24,018 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt +2025-01-21 21:11:24,019 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:11:24,071 INFO wandb-upload_4:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:24,142 INFO wandb-upload_3:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/kaiul4m6-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:11:24,343 INFO Thread-11 (_thread_body):2645977 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:11:24,343 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,343 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:11:24,343 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,343 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:11:24,343 INFO SenderThread:2645977 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:11:24,344 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:11:24,344 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,345 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:11:24,345 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,345 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:11:24,345 INFO SenderThread:2645977 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:11:24,580 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,633 INFO SenderThread:2645977 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:11:24,633 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:11:24,633 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,634 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,634 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:11:24,634 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,634 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send():379] send: final +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send():379] send: footer +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,635 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,636 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:11:24,637 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:11:24,637 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:11:24,638 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:24,688 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,688 INFO MainThread:2645977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:11:24,688 INFO MainThread:2645977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:11:24,689 INFO MainThread:2645977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:11:24,689 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:11:24,689 INFO HandlerThread:2645977 [handler.py:finish():882] shutting down handler +2025-01-21 21:11:25,636 INFO WriterThread:2645977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [sender.py:finish():1615] shutting down sender +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [file_pusher.py:join():175] waiting for file pusher diff --git a/bn7ul4i2/wandb/debug.log b/bn7ul4i2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..caecacfac9322f0558ebb953466c383d3761cb42 --- /dev/null +++ b/bn7ul4i2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Configure stats pid to 2645796 +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/settings +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug.log +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug-internal.log +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2'} +2025-01-21 21:06:51,384 INFO MainThread:2645796 [wandb_init.py:init():619] starting backend +2025-01-21 21:06:51,384 INFO MainThread:2645796 [wandb_init.py:init():623] setting up manager +2025-01-21 21:06:51,387 INFO MainThread:2645796 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:06:51,388 INFO MainThread:2645796 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:06:51,397 INFO MainThread:2645796 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:06:51,405 INFO MainThread:2645796 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:06:51,735 INFO MainThread:2645796 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:06:51,774 INFO MainThread:2645796 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:06:51,774 INFO MainThread:2645796 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:06:52,117 INFO MainThread:2645796 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:06:52,117 INFO MainThread:2645796 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/tb, True +2025-01-21 21:06:56,539 INFO MainThread:2645796 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:11:25,690 WARNING MsgRouterThr:2645796 [router.py:message_loop():77] message_loop has been closed diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc15b0744f2adb0591c7ce8e22f4aff3199bc510 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3465_0' +wandb_notes: + desc: null + value: 3465_0_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737511611 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8da10724284d70d8fca0dedf572670b3116372fc --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch @@ -0,0 +1,28 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..e77ad21 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -139,12 +139,12 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 06/20 demos per book + # crp_D+mask+act history +-# wandb_run_id: '3465_0' ++wandb_run_id: '3465_0' + # wandb_run_id: '3465_1' # seed 1 + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 + # wandb_run_id: '4295_2' # seed 2 + diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json new file mode 100644 index 0000000000000000000000000000000000000000..047e68504e394c796ca970017c4c5898d3c82cab --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[52, 55, 0.9454545454545454, 0.8353562056347568, 0.9903551748904372, 0.8640519752370386, 0.9883701744443953, 0.879273854120217, 0.9797626267159094, 0.8863567634644269, 0.9788690143238414]]} \ No newline at end of file diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1fdad059172460d96638ea3f1c4b3d1a35b60620 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92d3370f0db4ebdd4aaf2172036b99317be720ae0952524d557ebe3c439b662 +size 1171918 diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..02e238a1342946ee7d93faaa6d9d2b82cefe9dbb --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a38d84554f950c6b0497c97ea3f3390c361b7c3f4f4ddab98cbeee49507190cb +size 1185838 diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4bae28d2ebd4f7cfeb6f68d8fc0d62e4e0c4c422 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428c4186f8f49dcdf56485fbf404101bf90ec5646c06f2b4294019741ec3860a +size 1186953 diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f7d7d03ff15e06ba71309fc6c42b537ffb4670a9 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0cf70711762442af0f56c2c14d782a6116c3fda485ea86581a3951c50f3f6cf +size 1126837 diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c43ac9f62459bc29210e81346d9a35007c4de30b --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd715cc078221a229a9dc2bedfce4214f34bb940ca892bb65ed257dc339dc4c3 +size 1175795 diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6ee842f328c2fa5eafb8616af98592b979a25956 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737511616.540388]: resetting environment +[INFO] [1737511616.544609]: cleared current plan +[INFO] [1737511616.545434]: moving to home +[INFO] [1737511617.547647]: reached home +[INFO] [1737511617.548899]: reset action history +[INFO] [1737511620.257214]: environment reset +Starting episode 0 +[INFO] [1737511620.261200]: resetting environment +[INFO] [1737511620.268362]: cleared current plan +[INFO] [1737511620.269246]: moving to home +[INFO] [1737511621.272043]: reached home +[INFO] [1737511621.273525]: reset action history +[INFO] [1737511623.982186]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737511625.723917]: resetting environment +[INFO] [1737511625.730014]: cleared current plan +[INFO] [1737511625.731082]: moving to home +[INFO] [1737511626.733281]: reached home +[INFO] [1737511626.734806]: reset action history +[INFO] [1737511629.441945]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/episode_rosbags/episode_0_2025-01-21-21-07-09.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2645796.log +started roslaunch server http://158.130.50.37:40681/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2646091] +process[print_text-2]: started with pid [2646115] +process[rosbag_record-3]: started with pid [2646116] +started bagging! +For topic gripper_width: timestamp difference is 110279343 for nearest: 1737511632912503025 - target: 1737511632802223682 at idx 45 +[INFO] [1737511656.430577]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737511657.170595]: Stored episode 1. +[INFO] [1737511657.170957]: Saving video... +[INFO] [1737511657.360925]: Video saved! +Starting episode 1 +[INFO] [1737511657.375257]: resetting environment +[INFO] [1737511657.383374]: cleared current plan +[INFO] [1737511657.386830]: moving to home +[INFO] [1737511660.989263]: reached home +[INFO] [1737511660.990164]: reset action history +[INFO] [1737511663.696702]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511668.990849]: resetting environment +[INFO] [1737511668.993089]: cleared current plan +[INFO] [1737511668.993249]: moving to home +[INFO] [1737511669.994816]: reached home +[INFO] [1737511669.995220]: reset action history +[INFO] [1737511672.702000]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/episode_rosbags/episode_1_2025-01-21-21-07-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2645796.log +started roslaunch server http://158.130.50.37:44317/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2646224] +process[print_text-5]: started with pid [2646225] +process[rosbag_record-6]: started with pid [2646226] +started bagging! +[INFO] [1737511699.657525]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737511700.437499]: Stored episode 2. +[INFO] [1737511700.437755]: Saving video... +[INFO] [1737511700.946393]: Video saved! +Starting episode 2 +[INFO] [1737511700.976532]: resetting environment +[INFO] [1737511700.982657]: cleared current plan +[INFO] [1737511700.982837]: moving to home +[WARN] [1737511700.981773]: Plan exhausted +[INFO] [1737511704.585216]: reached home +[INFO] [1737511704.588433]: reset action history +[INFO] [1737511707.368458]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511714.089243]: resetting environment +[INFO] [1737511714.100786]: cleared current plan +[INFO] [1737511714.101033]: moving to home +[INFO] [1737511715.102616]: reached home +[INFO] [1737511715.103114]: reset action history +[INFO] [1737511717.810900]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/episode_rosbags/episode_2_2025-01-21-21-08-37.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2645796.log +started roslaunch server http://158.130.50.37:36511/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2646385] +process[print_text-8]: started with pid [2646386] +process[rosbag_record-9]: started with pid [2646387] +started bagging! +For topic gripper_width: timestamp difference is 122985174 for nearest: 1737511730912508299 - target: 1737511730789523125 at idx 165 +For topic gripper_width: timestamp difference is -107763679 for nearest: 1737511733912504761 - target: 1737511734020268440 at idx 203 +For topic gripper_width: timestamp difference is -103753492 for nearest: 1737511738912501887 - target: 1737511739016255379 at idx 249 +For topic gripper_width: timestamp difference is -101141826 for nearest: 1737511742312502965 - target: 1737511742413644791 at idx 249 +For topic gripper_width: timestamp difference is -100977051 for nearest: 1737511742512502086 - target: 1737511742613479137 at idx 249 +[INFO] [1737511745.081918]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1737511745.869116]: Stored episode 3. +[INFO] [1737511745.872715]: Saving video... +[INFO] [1737511746.217016]: Video saved! +Starting episode 3 +[INFO] [1737511746.265750]: resetting environment +[INFO] [1737511746.276907]: cleared current plan +[INFO] [1737511746.277231]: moving to home +[INFO] [1737511749.879480]: reached home +[INFO] [1737511749.892061]: reset action history +[INFO] [1737511752.730201]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511757.671679]: resetting environment +[INFO] [1737511757.689012]: cleared current plan +[INFO] [1737511757.691651]: moving to home +[INFO] [1737511758.693896]: reached home +[INFO] [1737511758.696450]: reset action history +[INFO] [1737511761.485044]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/episode_rosbags/episode_3_2025-01-21-21-09-21.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2645796.log +started roslaunch server http://158.130.50.37:45191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2646506] +process[print_text-11]: started with pid [2646507] +process[rosbag_record-12]: started with pid [2646531] +started bagging! +For topic gripper_width: timestamp difference is 101066146 for nearest: 1737511786179177079 - target: 1737511786078110933 at idx 249 +For topic gripper_width: timestamp difference is 101212406 for nearest: 1737511786379169607 - target: 1737511786277957201 at idx 249 +For topic gripper_width: timestamp difference is 101835977 for nearest: 1737511787179168950 - target: 1737511787077332973 at idx 249 +For topic gripper_width: timestamp difference is 102648119 for nearest: 1737511788179196219 - target: 1737511788076548100 at idx 249 +For topic gripper_width: timestamp difference is 102774621 for nearest: 1737511788379167558 - target: 1737511788276392937 at idx 249 +[INFO] [1737511788.564717]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1737511789.343598]: Stored episode 4. +[INFO] [1737511789.345188]: Saving video... +[WARN] [1737511789.893276]: Plan exhausted +[WARN] [1737511789.933298]: Plan exhausted +[WARN] [1737511789.974797]: Plan exhausted +[WARN] [1737511790.013627]: Plan exhausted +[WARN] [1737511790.053765]: Plan exhausted +[INFO] [1737511789.991468]: Video saved! +Starting episode 4 +[INFO] [1737511790.039965]: resetting environment +[INFO] [1737511790.063480]: cleared current plan +[INFO] [1737511790.068667]: moving to home +[INFO] [1737511793.478957]: reached home +[INFO] [1737511793.486034]: reset action history +[INFO] [1737511796.395450]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737511802.099564]: resetting environment +[INFO] [1737511802.117130]: cleared current plan +[INFO] [1737511802.120013]: moving to home +[INFO] [1737511803.128554]: reached home +[INFO] [1737511803.135347]: reset action history +[INFO] [1737511805.920209]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/episode_rosbags/episode_4_2025-01-21-21-10-05.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2645796.log +started roslaunch server http://158.130.50.37:35791/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2646675] +process[print_text-14]: started with pid [2646676] +process[rosbag_record-15]: started with pid [2646677] +started bagging! +For topic gripper_width: timestamp difference is -111872399 for nearest: 1737511811312508380 - target: 1737511811424380779 at idx 69 +[INFO] [1737511833.079415]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1737511833.861117]: Stored episode 5. +[INFO] [1737511833.869146]: Saving video... +[WARN] [1737511834.255630]: Plan exhausted +[INFO] [1737511834.182939]: Video saved! +[INFO] [1737511834.230549]: resetting environment +[INFO] [1737511834.261364]: cleared current plan +[INFO] [1737511834.261920]: moving to home +[INFO] [1737511837.866223]: reached home +[INFO] [1737511837.879003]: reset action history +[INFO] [1737511840.725943]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1c857bef8bbc64a5f0cb0a4a082f7d4dddc5287a --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:06:51.799700", + "startedAt": "2025-01-22T02:06:51.381170", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.5710000000004, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4497.405, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.988, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.469, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.983, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.951, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 344.01396560668945 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f159490c00b8cbd911806b4379d365570cfe038f --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "b92d3370f0db4ebdd4aaf2172036b99317be720ae0952524d557ebe3c439b662", "size": 1171918, "path": "media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4"}, "global_step": 663, "_timestamp": 1737511844.145754, "_runtime": 232.75743198394775, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "a38d84554f950c6b0497c97ea3f3390c361b7c3f4f4ddab98cbeee49507190cb", "size": 1185838, "path": "media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4"}, "eval/num_success": 4.0, "episode": 55.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "428c4186f8f49dcdf56485fbf404101bf90ec5646c06f2b4294019741ec3860a", "size": 1186953, "path": "media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f0cf70711762442af0f56c2c14d782a6116c3fda485ea86581a3951c50f3f6cf", "size": 1126837, "path": "media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cd715cc078221a229a9dc2bedfce4214f34bb940ca892bb65ed257dc339dc4c3", "size": 1175795, "path": "media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://j9pm84eet0my3xm8jop6t8wovja407h2cwqpw3klu2cf8l69cko06ee1gbnntrnwvw6qq4n9uxk4v8w6dyrossejgkv1bfpzlqexoutwle4wqllo7mf220w93ba6z5d2/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://awvvtcsjmhxz8xk3w8o7s9xanmtwazw35l0flku99rrjz8z5b43yygk2r11i15cx81fg3ewuhzyfomksbgu4qwgmynqlvcwl712rhd8h71ztv64trywyb6ezsmf6tidd:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 52, "total_num_episodes": 55, "total_success_rate": 0.9454545454545454, "total_success_rate_lb_0.01": 0.8353562056347568, "total_success_rate_ub_0.01": 0.9903551748904372, "total_success_rate_lb_0.025": 0.8640519752370386, "total_success_rate_ub_0.025": 0.9883701744443953, "total_success_rate_lb_0.05": 0.879273854120217, "total_success_rate_ub_0.05": 0.9797626267159094, "total_success_rate_lb_0.1": 0.8863567634644269, "total_success_rate_ub_0.1": 0.9788690143238414, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2033532332aee60c9249bb136a454fc35cacf6b9526157acbd6791ee7578f602", "size": 519, "artifact_path": "wandb-client-artifact://8sqit6ro0kto21oyucu2i1omhnddb013mcehvgpmnvo9g6w5w95salx2osf310i3rjj7gxkik8ngfgyzvz5332guy05x6bt22u3m1kebg9x9f0qileeoxcs1ok539gud/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://55imji2h4twy9dyqgpe8w9css0xehn4p2cnb9k1mt9rsmr6ku8u3yf30nnhek3dwi93kfpbj98xexenxox8xvppdnby9k69ths4m7vk1ucsc22zd6bnzflgfc25v6ka4:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 261}, "eval/total_success_rate": 0.9454545378684998, "eval/total_success_rate_lb_0.01": 0.8353561758995056, "eval/total_success_rate_ub_0.01": 0.9903551936149597, "eval/total_success_rate_lb_0.025": 0.8640519976615906, "eval/total_success_rate_ub_0.025": 0.9883701801300049, "eval/total_success_rate_lb_0.05": 0.8792738318443298, "eval/total_success_rate_ub_0.05": 0.979762613773346, "eval/total_success_rate_lb_0.1": 0.886356770992279, "eval/total_success_rate_ub_0.1": 0.9788690209388733} \ No newline at end of file diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug-internal.log b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6c2eae2b7bfd7c60c715cb32efd95510a06b4aa8 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug-internal.log @@ -0,0 +1,768 @@ +2025-01-21 21:06:51,388 INFO StreamThr :2645977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2645977, started at: 2025-01-21 21:06:51.388319 +2025-01-21 21:06:51,389 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:06:51,397 INFO WriterThread:2645977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb +2025-01-21 21:06:51,398 DEBUG SenderThread:2645977 [sender.py:send():379] send: header +2025-01-21 21:06:51,407 DEBUG SenderThread:2645977 [sender.py:send():379] send: run +2025-01-21 21:06:51,716 INFO SenderThread:2645977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files +2025-01-21 21:06:51,717 INFO SenderThread:2645977 [sender.py:_start_run_threads():1188] run started: bn7ul4i2 with start time 1737511611.388322 +2025-01-21 21:06:51,735 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:06:51,735 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:06:51,781 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:06:51,785 DEBUG HandlerThread:2645977 [system_info.py:__init__():26] System info init +2025-01-21 21:06:51,786 DEBUG HandlerThread:2645977 [system_info.py:__init__():41] System info init done +2025-01-21 21:06:51,786 INFO HandlerThread:2645977 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:06:51,786 INFO SystemMonitor:2645977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:06:51,786 INFO HandlerThread:2645977 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:06:51,787 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:06:51,788 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:06:51,788 INFO SystemMonitor:2645977 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:06:51,799 DEBUG HandlerThread:2645977 [system_info.py:probe():152] Probing system +2025-01-21 21:06:51,801 DEBUG HandlerThread:2645977 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_info.py:probe():200] Probing system done +2025-01-21 21:06:51,805 DEBUG HandlerThread:2645977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:06:51.799700', 'startedAt': '2025-01-22T02:06:51.381170', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.5710000000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4497.405, 'min': 800.0, 'max': 4700.0}, {'current': 4499.988, 'min': 800.0, 'max': 4700.0}, {'current': 4496.469, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.983, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.951, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 344.01396560668945}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:06:51,806 INFO HandlerThread:2645977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:06:51,806 INFO HandlerThread:2645977 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:06:51,806 DEBUG HandlerThread:2645977 [system_info.py:_save_code():44] Saving code +2025-01-21 21:06:51,807 DEBUG HandlerThread:2645977 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:06:51,807 DEBUG HandlerThread:2645977 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:06:51,869 DEBUG HandlerThread:2645977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:06:51,869 INFO HandlerThread:2645977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:06:51,872 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:06:51,872 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:06:51,872 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:06:51,873 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:06:52,114 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:06:52,114 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:06:52,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:06:52,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:06:52,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:52,117 INFO HandlerThread:2645977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/tb" + save: true +} +control { + relay_id: "8fcde4342c68447694e117753e76da0a" +} +_info { + stream_id: "bn7ul4i2" +} + +2025-01-21 21:06:52,117 DEBUG HandlerThread:2645977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:06:52,125 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/u8tdtv45-wandb-metadata.json +2025-01-21 21:06:52,143 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/3pw2oj2i-code/FISH/eval_robot.py +2025-01-21 21:06:52,180 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/dukk9114-diff.patch +2025-01-21 21:06:52,185 DEBUG SenderThread:2645977 [sender.py:send():379] send: telemetry +2025-01-21 21:06:52,185 DEBUG SenderThread:2645977 [sender.py:send():379] send: tbrecord +2025-01-21 21:06:52,186 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:06:52,186 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 with policy live +2025-01-21 21:06:52,388 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/8dovbc8h-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:52,718 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH +2025-01-21 21:06:52,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code +2025-01-21 21:06:53,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:54,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:54,717 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:55,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:56,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:56,540 DEBUG SenderThread:2645977 [sender.py:send():379] send: config +2025-01-21 21:06:56,540 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:06:57,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:58,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:06:58,719 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:06:59,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:00,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:01,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:02,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:02,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:02,720 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:03,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:04,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:05,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:06,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:06,721 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:07,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:07,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:07,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:07,277 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:08,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:08,721 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:09,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:10,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:10,722 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:11,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:12,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:12,723 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:12,913 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:13,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:14,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:14,723 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:15,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:16,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:17,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:18,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:18,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:19,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:20,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:21,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:22,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:22,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:22,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:23,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:23,701 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:24,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:24,726 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:07:25,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:26,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:27,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:28,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:29,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:29,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:30,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:31,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:32,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:33,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:34,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:35,116 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:35,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:36,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:37,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:37,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:37,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:37,405 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:07:37,406 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 with policy now +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:07:37,406 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:07:37,407 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:07:37,698 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/7seopqdc-media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:37,729 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:37,729 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:07:37,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media +2025-01-21 21:07:38,008 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/4cu6sdrr-media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:07:38,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:38,730 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:39,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:40,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:40,408 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:41,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:42,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:42,731 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:43,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:44,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:44,732 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:45,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:45,737 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:46,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:47,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:48,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:49,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:49,733 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:07:50,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:50,734 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:51,037 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:51,122 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:51,788 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:07:51,789 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:07:52,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:07:52,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:07:52,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:52,735 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:53,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:54,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:54,735 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:07:55,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:56,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:56,948 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:07:57,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:58,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:07:59,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:00,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:01,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:02,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:02,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:03,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:04,124 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:05,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:06,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:07,115 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:07,115 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:07,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:07,256 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:08,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:09,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:10,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:11,125 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:12,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:13,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:13,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:14,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:15,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:16,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:17,126 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:18,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:18,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:19,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:20,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:20,743 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:20,974 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:08:20,974 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 with policy now +2025-01-21 21:08:20,974 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:08:20,975 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:08:20,975 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:08:20,975 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:08:21,129 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:21,263 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/gvmll21x-media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:21,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:08:21,789 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:08:22,068 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/g0sx2ln3-media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:08:22,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:22,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:22,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:22,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:23,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:23,277 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:23,744 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:24,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:25,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:26,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:26,745 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:27,139 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:28,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:28,417 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:28,746 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:29,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:30,132 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:31,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:32,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:33,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:34,074 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:34,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:34,592 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:08:34,592 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:08:34,593 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:08:34,748 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:08:34,749 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:08:35,038 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/287aceph-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:08:35,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:36,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:36,749 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:37,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:37,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:37,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:38,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:38,750 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:39,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:39,476 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:40,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:40,751 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:41,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:42,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:43,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:44,135 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:45,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:45,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:46,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:47,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:48,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:49,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:50,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:50,921 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:51,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:51,790 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:08:52,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:08:52,117 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:08:52,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:52,755 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:53,136 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:54,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:55,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:56,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:56,150 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:08:56,756 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:08:57,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:58,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:08:59,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:00,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:00,757 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:01,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:02,121 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:02,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:03,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:04,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:04,759 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:05,139 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:06,141 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:06,265 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:09:06,266 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:09:06,266 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 with policy now +2025-01-21 21:09:06,266 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:06,267 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:06,267 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:06,550 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/w49ytuch-media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:06,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:09:07,102 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/3862svq1-media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:09:07,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:07,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:07,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:07,257 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:08,155 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:08,760 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:09,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:10,141 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:11,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:12,143 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:12,731 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:12,762 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:13,142 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:14,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:14,764 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:15,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:16,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:17,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:17,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:09:18,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:18,352 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:18,352 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:18,353 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:18,353 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:18,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:18,765 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:19,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:20,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:20,766 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:21,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:21,791 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:09:22,117 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:22,117 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:22,144 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:22,766 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:23,144 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:24,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:24,261 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:25,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:26,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:27,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:28,145 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:29,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:30,123 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:30,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:31,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:32,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:33,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:34,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:35,127 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:35,146 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:36,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:37,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:37,118 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:37,161 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:38,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:39,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:40,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:40,247 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:41,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:42,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:43,147 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:44,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:45,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:46,129 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:46,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:47,148 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:48,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:48,776 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:49,152 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:50,033 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:09:50,033 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 with policy now +2025-01-21 21:09:50,036 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:09:50,037 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:09:50,037 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:09:50,037 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:09:50,152 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:50,341 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/fwhbx8yt-media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:09:50,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:50,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:09:51,093 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/wpl0pvln-media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:09:51,159 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:51,777 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:51,791 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:09:51,792 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:52,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:09:52,133 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:09:52,177 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:52,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:53,164 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:54,163 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:54,778 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:55,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:56,166 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:57,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:09:57,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:58,178 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:09:58,780 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:09:59,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:00,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:01,180 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:02,138 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:02,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:02,407 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:02,407 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:02,408 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:02,781 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:02,781 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:03,062 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/y8fiahmw-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:03,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:04,178 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:04,782 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:05,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:06,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:06,783 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:07,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:07,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:07,172 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:07,204 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:08,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:08,783 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:09,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:10,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:11,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:12,173 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:12,785 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:13,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:13,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:14,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:15,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:16,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:17,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:18,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:18,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:19,174 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:20,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:21,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:21,792 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:10:22,118 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:22,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:22,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:23,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:23,272 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:24,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:25,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:26,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:27,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:28,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:29,131 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:29,175 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:30,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:31,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:32,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:33,176 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:34,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:34,183 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:34,229 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:34,229 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 with policy now +2025-01-21 21:10:34,230 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:34,230 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:34,230 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:34,231 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:34,517 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/7ygne6de-media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:34,792 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:34,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval +2025-01-21 21:10:35,086 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/kzqyng5g-media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:10:35,185 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:35,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:36,181 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:36,793 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:37,120 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:37,120 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:37,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:38,189 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:38,794 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:39,190 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:39,913 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:40,191 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:41,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:42,199 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:42,795 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:43,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:43,456 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:43,456 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:43,457 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:43,609 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:10:43,609 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval +2025-01-21 21:10:43,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table +2025-01-21 21:10:43,843 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:44,000 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/4f5g6jxq-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:44,134 INFO wandb-upload_0:2645977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmp6_2_p8 +2025-01-21 21:10:44,192 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:44,246 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:10:44,289 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:10:44,553 INFO SenderThread:2645977 [sender.py:send_request_log_artifact():1525] logged artifact run-bn7ul4i2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0OTgwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MjAxMQ==', 'latestArtifact': None}} +2025-01-21 21:10:44,553 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:44,553 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-21 21:10:44,553 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,554 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send():379] send: telemetry +2025-01-21 21:10:44,554 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,558 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,558 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,559 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,559 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,559 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,560 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,560 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,560 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,561 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,561 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,561 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,561 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,562 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,562 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:44,562 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:44,563 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:10:44,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:44,796 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:44,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval +2025-01-21 21:10:44,882 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/1bptd7ap-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:10:44,924 INFO wandb-upload_0:2645977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5qfs4pat +2025-01-21 21:10:45,015 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/rrq8xh32-media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:45,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:45,363 INFO SenderThread:2645977 [sender.py:send_request_log_artifact():1525] logged artifact run-bn7ul4i2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY0OTgyNQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5MjAxNg==', 'latestArtifact': None}} +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send():379] send: files +2025-01-21 21:10:45,363 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json with policy now +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:10:45,363 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:10:45,364 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:45,364 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:10:45,636 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/ahlofooo-media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:10:45,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:10:46,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:46,797 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:10:47,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:48,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:49,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:50,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:51,134 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:51,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:51,792 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:10:52,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:10:52,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:10:52,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:53,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:54,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:55,193 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:56,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:56,281 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:10:57,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:58,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:10:59,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:00,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:01,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:02,133 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:02,194 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:03,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:04,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:05,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:06,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:07,119 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:11:07,119 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:11:07,192 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:07,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:08,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:09,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:10,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:11,195 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:12,196 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:13,137 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:13,196 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:13,577 DEBUG SenderThread:2645977 [sender.py:send():379] send: exit +2025-01-21 21:11:13,577 INFO SenderThread:2645977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:11:13,578 INFO SenderThread:2645977 [sender.py:send_exit():588] handling runtime: 261 +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:send_exit():594] send defer +2025-01-21 21:11:13,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,579 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:11:13,579 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:11:13,579 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:11:13,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,579 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:11:13,580 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,580 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:11:13,580 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:11:13,580 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:11:13,580 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:11:13,580 DEBUG SystemMonitor:2645977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:11:13,580 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:11:13,651 INFO HandlerThread:2645977 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:11:13,651 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:11:13,652 DEBUG SenderThread:2645977 [sender.py:send():379] send: stats +2025-01-21 21:11:13,652 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,652 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:11:13,652 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:11:13,652 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:11:13,652 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:13,652 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:11:13,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:11:13,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:14,808 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:20,466 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,467 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:11:20,467 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send():379] send: history +2025-01-21 21:11:20,467 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,467 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:11:20,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:11:20,468 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:20,468 DEBUG SenderThread:2645977 [sender.py:send():379] send: summary +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:11:20,472 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:11:20,472 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,473 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:11:20,473 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:11:20,473 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:20,473 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:11:20,473 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:20,473 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:11:20,578 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:20,811 INFO Thread-12 :2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:23,149 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:11:23,149 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:23,149 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,150 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:11:23,150 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,150 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:11:23,150 INFO SenderThread:2645977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:11:23,152 INFO SenderThread:2645977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:11:23,154 INFO SenderThread:2645977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:11:23,169 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:11:23,169 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,169 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:11:23,169 DEBUG SenderThread:2645977 [sender.py:send():379] send: artifact +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzYyNTUzOQ==', 'versionIndex': 454}}} +2025-01-21 21:11:23,467 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:11:23,467 INFO SenderThread:2645977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:11:23,579 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml config.yaml +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json wandb-summary.json +2025-01-21 21:11:23,812 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt requirements.txt +2025-01-21 21:11:23,813 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:11:23,813 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:11:23,817 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log output.log +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/diff.patch diff.patch +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 media/videos/eval/0_eval_0_b92d3370f0db4ebdd4aa.mp4 +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 media/videos/eval/3_eval_3_f0cf70711762442af0f5.mp4 +2025-01-21 21:11:23,818 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 media/videos/eval/4_eval_4_cd715cc078221a229a9d.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 media/videos/eval/1_eval_1_a38d84554f950c6b0497.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 media/videos/eval/2_eval_2_428c4186f8f49dcdf564.mp4 +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json media/table/eval/total_success_rate_ci_6_2033532332aee60c9249.table.json +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-21 21:11:23,820 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:11:23,820 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:23,822 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:23,822 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:11:23,822 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:23,822 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:11:23,822 INFO SenderThread:2645977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:11:24,003 INFO wandb-upload_0:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/wandb-summary.json +2025-01-21 21:11:24,018 INFO wandb-upload_1:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/requirements.txt +2025-01-21 21:11:24,019 INFO wandb-upload_2:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/config.yaml +2025-01-21 21:11:24,071 INFO wandb-upload_4:2645977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/files/output.log +2025-01-21 21:11:24,142 INFO wandb-upload_3:2645977 [upload_job.py:push():130] Uploaded file /tmp/tmp5yjpo8npwandb/kaiul4m6-events.out.tfevents.1737511612.leonmkim-ROG-Strix-G15CS-G15CS.2645796.0 +2025-01-21 21:11:24,343 INFO Thread-11 (_thread_body):2645977 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:11:24,343 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,343 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:11:24,343 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,343 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:11:24,343 INFO SenderThread:2645977 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:11:24,344 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:11:24,344 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,345 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:11:24,345 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,345 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:11:24,345 INFO SenderThread:2645977 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:11:24,580 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,633 INFO SenderThread:2645977 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:11:24,633 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:11:24,633 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,634 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,634 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:11:24,634 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:11:24,634 INFO HandlerThread:2645977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send():379] send: final +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send():379] send: footer +2025-01-21 21:11:24,634 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: defer +2025-01-21 21:11:24,634 INFO SenderThread:2645977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,635 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:11:24,635 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:11:24,636 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:11:24,637 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:11:24,637 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:11:24,638 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:11:24,688 DEBUG SenderThread:2645977 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:11:24,688 INFO MainThread:2645977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:11:24,688 INFO MainThread:2645977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:11:24,689 INFO MainThread:2645977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:11:24,689 DEBUG HandlerThread:2645977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:11:24,689 INFO HandlerThread:2645977 [handler.py:finish():882] shutting down handler +2025-01-21 21:11:25,636 INFO WriterThread:2645977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [sender.py:finish():1615] shutting down sender +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:11:25,688 INFO SenderThread:2645977 [file_pusher.py:join():175] waiting for file pusher diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug.log b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..caecacfac9322f0558ebb953466c383d3761cb42 --- /dev/null +++ b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Configure stats pid to 2645796 +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/settings +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug.log +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/logs/debug-internal.log +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:06:51,383 INFO MainThread:2645796 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3465_0', 'wandb_notes': '3465_0_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2'} +2025-01-21 21:06:51,384 INFO MainThread:2645796 [wandb_init.py:init():619] starting backend +2025-01-21 21:06:51,384 INFO MainThread:2645796 [wandb_init.py:init():623] setting up manager +2025-01-21 21:06:51,387 INFO MainThread:2645796 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:06:51,388 INFO MainThread:2645796 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:06:51,397 INFO MainThread:2645796 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:06:51,405 INFO MainThread:2645796 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:06:51,735 INFO MainThread:2645796 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:06:51,774 INFO MainThread:2645796 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:06:51,774 INFO MainThread:2645796 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:06:52,115 INFO MainThread:2645796 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:06:52,117 INFO MainThread:2645796 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:06:52,117 INFO MainThread:2645796 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/bn7ul4i2/tb, True +2025-01-21 21:06:56,539 INFO MainThread:2645796 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:11:25,690 WARNING MsgRouterThr:2645796 [router.py:message_loop():77] message_loop has been closed diff --git a/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2e65d27283b0847d2e39bad6479dc29db2417fb1 Binary files /dev/null and b/bn7ul4i2/wandb/run-20250121_210651-bn7ul4i2/run-bn7ul4i2.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96115ed71dbd9badeba1698d3fd4d38af87810c9 --- /dev/null +++ b/config.yaml @@ -0,0 +1,573 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '3465_0' +wandb_notes: 3465_0_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_1_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/lwd6ufn0/.hydra/config.yaml b/lwd6ufn0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f829134d57914397f938447dc055c8c1721578e --- /dev/null +++ b/lwd6ufn0/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/lwd6ufn0/.hydra/hydra.yaml b/lwd6ufn0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e478d5dedcf0223d5510032d832a0424dbeab47 --- /dev/null +++ b/lwd6ufn0/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/lwd6ufn0 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/lwd6ufn0/.hydra/overrides.yaml b/lwd6ufn0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/lwd6ufn0/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/lwd6ufn0/eval_policy.log b/lwd6ufn0/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..f0305281a4976096fc1b1b610494abeb61083ff3 --- /dev/null +++ b/lwd6ufn0/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 20:46:12,524][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 20:46:12,527][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 20:46:16,270][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/mo0b6mh9/episode_rosbags/aligned_depth_to_color_K.npy b/mo0b6mh9/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/mo0b6mh9/episode_rosbags/cam_tf_world.npy b/mo0b6mh9/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/mo0b6mh9/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/mo0b6mh9/episode_rosbags/depth_K.npy b/mo0b6mh9/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/mo0b6mh9/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag b/mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..dad335c91ac961a9b6e71b744f207f123fb08475 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_13_2025-01-18-20-06-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a666e2844ad58d4c6c41be205ac82085bcc01975a45a47b9f87b575f0f47b6 +size 1412522746 diff --git a/mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag b/mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag new file mode 100644 index 0000000000000000000000000000000000000000..2a1e463e4ca906122b901ded2c86ed5853d89dbd --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_15_2025-01-18-20-07-44.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c687bbb50a784207405c8329867d2f1a6399956fc9f57a5a9555e8931a13d902 +size 1413581814 diff --git a/mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag b/mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..568fa54bd273681ce1f8fcf867df496035dc6754 --- /dev/null +++ b/mo0b6mh9/episode_rosbags/episode_18_2025-01-18-20-10-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3baa8abcfdc3ad7c32f4b885d8498598221ede58c9056ab9235a55db0d0104ca +size 1412848774 diff --git a/mo0b6mh9/eval_robot.log b/mo0b6mh9/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..777f634bc04ae01fc1523ea8841b42ef3a50de7d --- /dev/null +++ b/mo0b6mh9/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-18 19:54:51,361][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 19:54:51,364][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/snapshot_10249.pt b/snapshot_10249.pt new file mode 100644 index 0000000000000000000000000000000000000000..2365f2ed8c9bcd1d6299e158bd5444d91356d9cb --- /dev/null +++ b/snapshot_10249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:effac819374ec86fc7423c6557c8a576e15f7daf44d61609e6e5cf83f79814cb +size 909989782 diff --git a/snapshot_10499.pt b/snapshot_10499.pt new file mode 100644 index 0000000000000000000000000000000000000000..cba343a0636eb14db49037aea2163b317ec86159 --- /dev/null +++ b/snapshot_10499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ebd813fda69de8874f3e9060e3409a36a26dcb8a7a9a5fc9230ddb6540f6a8b +size 909989782 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6aa6a9c1435b195f3378812f73560c6329a0490 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6278f6da63e2283ac72e5279b982388501c7dec4aa3252005e3a57bffb70d17f +size 909989782 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b24acbc97da48b77b9f7dbe8f22c1a9907ab9e6 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79501bfbc39a6d3a900e43a331874d0be95e55dcf7360ae265c7d802d7edf055 +size 909989782 diff --git a/snapshot_12249.pt b/snapshot_12249.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0f57ef6dc2bfa143cdf56d1cebdc34d50ce1aab --- /dev/null +++ b/snapshot_12249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ac8e004a31c9b0584212f886a1187cd43eca20f41a1ec605fa552d8dc007de +size 909989782 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..01850ee8936568af60c96b0c994192ba4f4845a4 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7e11dbb3e8cb6ef189782060a5d30b9755d10b4e7bd3ec383b5b249e3bb151c +size 909989782 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c61dc85c0b66abadb10f46c58ff75262ebef3926 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b55ce590ab5d07c2c39dcc84689e008de9597ab373022323655f313a716b69 +size 909989782 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..34a792988aaf9027cd6fe0d3961e1da740a67076 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6afb2cf7a1815229ab03b6ed6872950993feb0a0da57be0427ebc2a645754581 +size 909989272 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1f66e136a8682928578e62ce8d4c772ab1b05bc --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2772b1ca845c30772a08edaa7a5ba03a6a57732c544ca8b788afd44e19f8032 +size 909989272 diff --git a/snapshot_6749.pt b/snapshot_6749.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6dafeb6c68beae469cd7c97ba68652339b1db0d --- /dev/null +++ b/snapshot_6749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a01207d832b3a5bbb344e234e576f825ba841c949dd61f309f14ea0fb2a3ed4 +size 909989272 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f8c5fa177d0baff07f9ef0840f3a7d3d3e3a45a --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aff8c75838d92061f49fef88b81704a22c27182d55df821992de39c83cba8daa +size 909989272 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce41416983df06b38a9642dd5cfd7212ef821490 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d73c7399877f000837e1567694870145ec11c0871fe8572a2481e43f479954 +size 909989782 diff --git a/snapshot_9749.pt b/snapshot_9749.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbff2dd598ba315147b287752661681d5adffa49 --- /dev/null +++ b/snapshot_9749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f495930868419394dbcf3a6514c7dc79f481ced72c457215ca7736891b4300af +size 909989782 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..c61dc85c0b66abadb10f46c58ff75262ebef3926 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b55ce590ab5d07c2c39dcc84689e008de9597ab373022323655f313a716b69 +size 909989782 diff --git a/tih68zef/.hydra/config.yaml b/tih68zef/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd7a0d9043cd058869b33cb503156d329d779fab --- /dev/null +++ b/tih68zef/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3465_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/tih68zef/.hydra/hydra.yaml b/tih68zef/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b66d5068a9d859d5d6f8a4cbf877c0958f4a6ee --- /dev/null +++ b/tih68zef/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3465_0/tih68zef + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/tih68zef/.hydra/overrides.yaml b/tih68zef/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/tih68zef/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/tih68zef/eval_policy.log b/tih68zef/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..7ac2cc2fa30f3fa5ef72224ed4a4e748d9ff3c06 --- /dev/null +++ b/tih68zef/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-18 18:47:37,206][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-18 18:47:37,209][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-18 18:47:39,665][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +